diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..aa5f11534259 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,15 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + + +4.22.0 > 4.23.0: + * Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework + and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary + pool can now be delivered to KVM hypervisors over NVMe-TCP instead of + Fibre Channel by setting transport=nvme-tcp on the pool's provider URL. + Volumes are identified on the host via EUI-128 NGUIDs and attached to + guests as plain block devices through the native NVMe multipath layer; + no device-mapper multipath configuration is required. A new + Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase / + NVMeTCPAdapter on the KVM side back the new pool type. diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 5b3e97698fda..e2b0bbd0b670 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -183,7 +183,8 @@ public static enum StoragePoolType { Linstor(true, true, EncryptionSupport.Storage), DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters StorPool(true, true, EncryptionSupport.Hypervisor), - FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.) private final boolean shared; private final boolean overProvisioning; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 6e03b84d20cf..b6d7522a6926 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -376,7 +376,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { StoragePoolType.RBD, StoragePoolType.PowerFlex, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(primaryPool.getType())) { + StoragePoolType.FiberChannel, + StoragePoolType.NVMeTCP).contains(primaryPool.getType())) { newTemplate.setFormat(ImageFormat.RAW); } else { newTemplate.setFormat(ImageFormat.QCOW2); @@ -409,7 +410,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { String path = null; - if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { + if (primaryStore.getPoolType() == StoragePoolType.FiberChannel + || primaryStore.getPoolType() == StoragePoolType.NVMeTCP) { path = destData.getPath(); } else { path = details != null ? details.get("managedStoreTarget") : null; @@ -3175,7 +3177,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) { StoragePoolType.RBD, StoragePoolType.PowerFlex, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(poolType)) { + StoragePoolType.FiberChannel, + StoragePoolType.NVMeTCP).contains(poolType)) { return ImageFormat.RAW; } else { return ImageFormat.QCOW2; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java new file mode 100644 index 000000000000..e37906cde2e2 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java @@ -0,0 +1,462 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; + +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Base class for KVM storage adapters that surface remote block volumes over + * NVMe-over-Fabrics (NVMe-oF). It is the NVMe-oF counterpart of + * {@link MultipathSCSIAdapterBase}: it does not drive device-mapper multipath + * and does not rescan the SCSI bus, because NVMe-oF has its own multipath + * (the kernel's native NVMe multipath) and namespaces show up via + * asynchronous event notifications as soon as the target grants access. + * + * Volumes are identified on the host by their EUI-128 NGUID, which udev + * exposes as {@code /dev/disk/by-id/nvme-eui.}. + */ +public abstract class MultipathNVMeOFAdapterBase implements StorageAdaptor { + protected static Logger LOGGER = LogManager.getLogger(MultipathNVMeOFAdapterBase.class); + static final Map MapStorageUuidToStoragePool = new HashMap<>(); + + static final int DEFAULT_DISK_WAIT_SECS = 240; + static final long NS_RESCAN_TIMEOUT_SECS = 5; + private static final long POLL_INTERVAL_MS = 2000; + + @Override + public KVMStoragePool getStoragePool(String uuid) { + KVMStoragePool pool = MapStorageUuidToStoragePool.get(uuid); + if (pool == null) { + // Dummy pool - adapters that dispatch per-volume don't need + // connectivity information on the pool itself. + pool = new MultipathNVMeOFPool(uuid, this); + MapStorageUuidToStoragePool.put(uuid, pool); + } + return pool; + } + + @Override + public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { + return getStoragePool(uuid); + } + + public abstract String getName(); + + @Override + public abstract Storage.StoragePoolType getStoragePoolType(); + + public abstract boolean isStoragePoolTypeSupported(Storage.StoragePoolType type); + + /** + * Parse a {@code type=NVMETCP; address=; connid.=; ...} + * volume path and produce an {@link AddressInfo} with the host-side device + * path set to {@code /dev/disk/by-id/nvme-eui.}. + */ + public AddressInfo parseAndValidatePath(String inPath) { + String type = null; + String address = null; + String connectionId = null; + String path = null; + String hostname = resolveHostnameShort(); + String hostnameFq = resolveHostnameFq(); + String[] parts = inPath.split(";"); + for (String part : parts) { + String[] pair = part.split("="); + if (pair.length != 2) { + continue; + } + String key = pair[0].trim(); + String value = pair[1].trim(); + if (key.equals("type")) { + type = value.toUpperCase(); + } else if (key.equals("address")) { + address = value; + } else if (key.equals("connid")) { + connectionId = value; + } else if (key.startsWith("connid.")) { + String inHostname = key.substring("connid.".length()); + if (inHostname.equals(hostname) || inHostname.equals(hostnameFq)) { + connectionId = value; + } + } + } + + if (!"NVMETCP".equals(type)) { + throw new CloudRuntimeException("Invalid address type provided for NVMe-oF target disk: " + type); + } + if (address == null) { + throw new CloudRuntimeException("NVMe-oF volume path is missing the required address field"); + } + path = "/dev/disk/by-id/nvme-eui." + address.toLowerCase(); + return new AddressInfo(type, address, connectionId, path); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumePath, KVMStoragePool pool) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to get physical disk, volume path or pool not specified"); + return null; + } + return getPhysicalDisk(parseAndValidatePath(volumePath), pool); + } + + private KVMPhysicalDisk getPhysicalDisk(AddressInfo address, KVMStoragePool pool) { + KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); + disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + + if (!isConnected(address.getPath())) { + if (!connectPhysicalDisk(address, pool, null)) { + throw new CloudRuntimeException("Unable to connect to NVMe namespace at " + address.getPath()); + } + } + long diskSize = getPhysicalDiskSize(address.getPath()); + disk.setSize(diskSize); + disk.setVirtualSize(diskSize); + return disk; + } + + @Override + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { + LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %d, %s, %s)", uuid, host, port, path, type)); + MultipathNVMeOFPool pool = new MultipathNVMeOFPool(uuid, host, port, path, type, details, this); + MapStorageUuidToStoragePool.put(uuid, pool); + return pool; + } + + @Override + public boolean deleteStoragePool(String uuid) { + MapStorageUuidToStoragePool.remove(uuid); + return true; + } + + @Override + public boolean deleteStoragePool(KVMStoragePool pool) { + return deleteStoragePool(pool.getUuid()); + } + + @Override + public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map details, boolean isVMMigrate) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to connect NVMe-oF physical disk: insufficient arguments"); + return false; + } + return connectPhysicalDisk(parseAndValidatePath(volumePath), pool, details); + } + + private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map details) { + if (address.getConnectionId() == null) { + LOGGER.error("NVMe-oF volume " + address.getPath() + " on pool " + pool.getUuid() + " is missing a connid. token in its path"); + return false; + } + long waitSecs = DEFAULT_DISK_WAIT_SECS; + if (details != null && details.containsKey(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString())) { + String waitTime = details.get(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString()); + if (StringUtils.isNotEmpty(waitTime)) { + try { + waitSecs = Integer.parseInt(waitTime); + } catch (NumberFormatException e) { + LOGGER.warn("Ignoring non-numeric " + com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString() + + "=[" + waitTime + "] on pool " + pool.getUuid() + ", falling back to default " + + DEFAULT_DISK_WAIT_SECS + "s"); + } + } + } + return waitForNamespace(address, pool, waitSecs); + } + + /** + * Poll for the EUI-keyed udev symlink to show up. On every iteration also + * nudge the kernel with {@code nvme ns-rescan} on every local NVMe + * controller, to cover arrays / firmware combinations that do not emit a + * reliable asynchronous event notification when a new namespace is + * mapped. + */ + private boolean waitForNamespace(AddressInfo address, KVMStoragePool pool, long waitSecs) { + if (waitSecs < 60) { + waitSecs = 60; + } + long deadline = System.currentTimeMillis() + (waitSecs * 1000); + File dev = new File(address.getPath()); + while (System.currentTimeMillis() < deadline) { + if (dev.exists() && isConnected(address.getPath())) { + long size = getPhysicalDiskSize(address.getPath()); + if (size > 0) { + LOGGER.debug("Found NVMe namespace at " + address.getPath()); + return true; + } + } + rescanAllControllers(); + try { + Thread.sleep(POLL_INTERVAL_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + LOGGER.debug("NVMe namespace did not appear at " + address.getPath() + " within " + waitSecs + "s"); + return false; + } + + private void rescanAllControllers() { + try { + File sysClass = new File("/sys/class/nvme"); + File[] ctrls = sysClass.listFiles(); + if (ctrls == null) { + return; + } + for (File ctrl : ctrls) { + Process p = new ProcessBuilder("nvme", "ns-rescan", "/dev/" + ctrl.getName()) + .redirectErrorStream(true).start(); + if (!p.waitFor(NS_RESCAN_TIMEOUT_SECS, TimeUnit.SECONDS)) { + // Kill runaway nvme-cli invocations so they do not pile + // up under the JVM on every poll iteration while we + // are still waiting for the namespace to appear. + LOGGER.debug("nvme ns-rescan /dev/" + ctrl.getName() + + " did not complete within " + NS_RESCAN_TIMEOUT_SECS + + "s; terminating"); + p.destroyForcibly(); + } + } + } catch (Exception e) { + LOGGER.debug("nvme ns-rescan attempt failed: " + e.getMessage()); + } + } + + @Override + public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { + // NVMe-oF: the kernel drops the namespace as soon as the target + // removes the host(-group) connection. No host-side action needed. + return true; + } + + @Override + public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { + return true; + } + + @Override + public boolean disconnectPhysicalDiskByPath(String localPath) { + // Same rationale as disconnectPhysicalDisk above. Only claim paths + // that look like NVMe EUI symlinks so we don't swallow foreign paths. + return localPath != null && localPath.startsWith("/dev/disk/by-id/nvme-eui."); + } + + @Override + public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format) { + throw new UnsupportedOperationException("Deletion of NVMe namespaces is the storage provider's responsibility"); + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, + Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'"); + } + + @Override + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, QemuImg.PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + throw new UnsupportedOperationException("Unimplemented method 'createTemplateFromDisk'"); + } + + @Override + public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { + throw new UnsupportedOperationException("Unimplemented method 'listPhysicalDisks'"); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { + return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); + } + + /** + * Copy a template or source disk into a pre-provisioned NVMe namespace on + * this pool, so it can be consumed by a VM as a root or data volume. + * + * The destination namespace is expected to have already been created on + * the storage provider and connected to this host's hostgroup (that is + * the storage orchestrator's responsibility, not the KVM adapter's). All + * this method does is resolve the destination device path via + * {@link #getPhysicalDisk} - which will nvme ns-rescan and wait for the + * by-id/nvme-eui.<NGUID> symlink to show up if the kernel has not + * picked it up yet - and {@code qemu-img convert} the source image into + * the raw block device. + * + * User-space encryption passphrases are not supported: the provider + * already encrypts at rest and qemu-img LUKS on top of a shared + * hostgroup-scoped namespace is not a sensible layering. + */ + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { + if (disk == null || StringUtils.isEmpty(name) || destPool == null) { + throw new CloudRuntimeException("Unable to copy disk to NVMe-oF pool: source disk, destination volume name or destination pool not specified"); + } + if (srcPassphrase != null || destPassphrase != null) { + throw new CloudRuntimeException("NVMe-oF adapter does not support user-space encrypted source or destination volumes"); + } + + KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(name); + if (destDisk == null || StringUtils.isEmpty(destDisk.getPath())) { + throw new CloudRuntimeException("Unable to resolve NVMe namespace for destination volume [" + name + "] on pool [" + destPool.getUuid() + "]"); + } + + destDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + destDisk.setVirtualSize(disk.getVirtualSize()); + destDisk.setSize(disk.getSize()); + + LOGGER.info(String.format("Copying source disk [path=%s, format=%s, virtualSize=%d] to NVMe-oF namespace [path=%s] on pool [%s]", + disk.getPath(), disk.getFormat(), disk.getVirtualSize(), destDisk.getPath(), destPool.getUuid())); + + QemuImgFile srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + + try { + QemuImg qemu = new QemuImg(timeout); + qemu.convert(srcFile, destFile, true); + } catch (QemuImgException | LibvirtException e) { + throw new CloudRuntimeException("Failed to copy source disk [" + disk.getPath() + "] to NVMe-oF namespace [" + + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]: " + e.getMessage(), e); + } + + LOGGER.info("Successfully copied source disk to NVMe-oF namespace [" + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]"); + return destDisk; + } + + @Override + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplate'"); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplateBacking'"); + } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + throw new UnsupportedOperationException("Unimplemented method 'createTemplateFromDirectDownloadFile'"); + } + + @Override + public boolean refresh(KVMStoragePool pool) { + return true; + } + + @Override + public boolean createFolder(String uuid, String path) { + throw new UnsupportedOperationException("Unimplemented method 'createFolder'"); + } + + @Override + public boolean createFolder(String uuid, String path, String localPath) { + throw new UnsupportedOperationException("Unimplemented method 'createFolder'"); + } + + public void resize(String path, String vmName, long newSize) { + throw new UnsupportedOperationException("Volume resize on NVMe-oF pools is driven by the storage provider, not the KVM adapter"); + } + + boolean isConnected(String path) { + Script test = new Script("/bin/test", LOGGER); + test.add("-b", path); + test.execute(); + return test.getExitValue() == 0; + } + + long getPhysicalDiskSize(String diskPath) { + if (StringUtils.isEmpty(diskPath)) { + return 0; + } + Script cmd = new Script("blockdev", LOGGER); + cmd.add("--getsize64", diskPath); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = cmd.execute(parser); + if (result != null) { + LOGGER.debug("Unable to get the disk size at path: " + diskPath); + return 0; + } + try { + return Long.parseLong(parser.getLine()); + } catch (NumberFormatException e) { + return 0; + } + } + + private static String resolveHostnameShort() { + try { + String h = java.net.InetAddress.getLocalHost().getHostName(); + int dot = h.indexOf('.'); + return dot > 0 ? h.substring(0, dot) : h; + } catch (Exception e) { + return null; + } + } + + private static String resolveHostnameFq() { + try { + return java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } catch (Exception e) { + return null; + } + } + + /** + * Same shape as {@link MultipathSCSIAdapterBase.AddressInfo}. Kept + * separate so this class can be consumed by adapters that don't share the + * SCSI base. + */ + public static final class AddressInfo { + String type; + String address; + String connectionId; + String path; + + public AddressInfo(String type, String address, String connectionId, String path) { + this.type = type; + this.address = address; + this.connectionId = connectionId; + this.path = path; + } + + public String getType() { return type; } + public String getAddress() { return address; } + public String getConnectionId() { return connectionId; } + public String getPath() { return path; } + + public String toString() { + return String.format("AddressInfo %s [address=%s, connectionId=%s, path=%s]", type, address, connectionId, path); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java new file mode 100644 index 000000000000..581762e388a5 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.joda.time.Duration; + +import com.cloud.agent.api.to.HostTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ProvisioningType; + +/** + * KVMStoragePool for NVMe-over-Fabrics pools. Mirror of + * {@link MultipathSCSIPool} for adapters based on + * {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated + * back to the adapter; the pool itself only tracks addressing/identity. + */ +public class MultipathNVMeOFPool implements KVMStoragePool { + private final String uuid; + private final String sourceHost; + private final int sourcePort; + private final String sourceDir; + private final Storage.StoragePoolType storagePoolType; + private final StorageAdaptor storageAdaptor; + private final Map details; + private long capacity; + private long used; + private long available; + + public MultipathNVMeOFPool(String uuid, String host, int port, String path, + Storage.StoragePoolType poolType, Map poolDetails, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = host; + this.sourcePort = port; + this.sourceDir = path; + this.storagePoolType = poolType; + this.storageAdaptor = adaptor; + this.details = poolDetails; + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = null; + this.sourcePort = -1; + this.sourceDir = null; + this.storagePoolType = Storage.StoragePoolType.NVMeTCP; + this.storageAdaptor = adaptor; + this.details = new HashMap<>(); + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public boolean connectPhysicalDisk(String volumeUuid, Map details) { + return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumeId) { + return storageAdaptor.getPhysicalDisk(volumeId, this); + } + + @Override + public boolean disconnectPhysicalDisk(String volumeUuid) { + return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this); + } + + @Override + public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) { + return true; + } + + @Override + public List listPhysicalDisks() { + return null; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setCapacity(long capacity) { this.capacity = capacity; } + @Override public long getCapacity() { return this.capacity; } + public void setUsed(long used) { this.used = used; } + @Override public long getUsed() { return this.used; } + public void setAvailable(long available) { this.available = available; } + @Override public long getAvailable() { return this.available; } + + @Override public boolean refresh() { return false; } + @Override public boolean isExternalSnapshot() { return true; } + @Override public String getLocalPath() { return null; } + @Override public String getSourceHost() { return this.sourceHost; } + @Override public String getSourceDir() { return this.sourceDir; } + @Override public int getSourcePort() { return this.sourcePort; } + @Override public String getAuthUserName() { return null; } + @Override public String getAuthSecret() { return null; } + @Override public Storage.StoragePoolType getType() { return storagePoolType; } + @Override public boolean delete() { return false; } + @Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; } + @Override public boolean createFolder(String path) { return false; } + @Override public boolean supportsConfigDriveIso() { return false; } + @Override public Map getDetails() { return this.details; } + @Override public boolean isPoolSupportHA() { return false; } + @Override public String getHearthBeatPath() { return null; } + + @Override + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + return null; + } + + @Override public String getStorageNodeId() { return null; } + + @Override + public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { return null; } + + @Override + public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java new file mode 100644 index 000000000000..596f5d7bb9c5 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import com.cloud.storage.Storage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type. + * All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this + * class just binds that logic to a pool type so + * {@link KVMStoragePoolManager} can find it via reflection. + */ +public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase { + private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class); + + public NVMeTCPAdapter() { + LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer"); + } + + @Override + public String getName() { + return "NVMeTCPAdapter"; + } + + @Override + public Storage.StoragePoolType getStoragePoolType() { + return Storage.StoragePoolType.NVMeTCP; + } + + @Override + public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) { + return Storage.StoragePoolType.NVMeTCP.equals(type); + } +} diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java index 25577903e3d8..a954560508c4 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java @@ -35,6 +35,7 @@ public interface ProviderVolume { public String getExternalName(); public String getExternalConnectionId(); public enum AddressType { - FIBERWWN + FIBERWWN, + NVMETCP } } diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java index 771f79887e0f..42c832ec1e4d 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java @@ -179,7 +179,7 @@ public DataStore initialize(Map dsInfos) { parameters.setHost(uri.getHost()); parameters.setPort(uri.getPort()); parameters.setPath(uri.getPath() + "?" + uri.getQuery()); - parameters.setType(StoragePoolType.FiberChannel); + parameters.setType(pickPoolType(uri)); parameters.setZoneId(zoneId); parameters.setPodId(podId); parameters.setClusterId(clusterId); @@ -401,4 +401,26 @@ public void disableStoragePool(DataStore store) { logger.info("Disabling storage pool {}", store); _dataStoreHelper.disable(store); } + + /** + * Resolve the CloudStack StoragePoolType from the provider URL. Adaptive + * plugins advertise the underlying fabric via a {@code transport=} query + * parameter on the URL; when absent we keep the legacy FiberChannel + * default for backwards compatibility with adapters that still assume it. + */ + private static StoragePoolType pickPoolType(java.net.URL uri) { + String query = uri.getQuery(); + if (query != null) { + for (String tok : query.split("&")) { + int i = tok.indexOf('='); + if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) { + String value = tok.substring(i + 1); + if ("nvme-tcp".equalsIgnoreCase(value)) { + return StoragePoolType.NVMeTCP; + } + } + } + } + return StoragePoolType.FiberChannel; + } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java index 41125f3e1135..3a58d9f806d1 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java @@ -73,6 +73,9 @@ public class FlashArrayAdapter implements ProviderAdapter { public static final String HOSTGROUP = "hostgroup"; public static final String STORAGE_POD = "pod"; + public static final String TRANSPORT = "transport"; + public static final String TRANSPORT_FC = "fc"; + public static final String TRANSPORT_NVME_TCP = "nvme-tcp"; public static final String KEY_TTL = "keyttl"; public static final String CONNECT_TIMEOUT_MS = "connectTimeoutMs"; public static final String POST_COPY_WAIT_MS = "postCopyWaitMs"; @@ -88,6 +91,7 @@ public class FlashArrayAdapter implements ProviderAdapter { static final ObjectMapper mapper = new ObjectMapper(); public String pod = null; public String hostgroup = null; + private AddressType volumeAddressType = AddressType.FIBERWWN; private String username; private String password; private String accessToken; @@ -121,7 +125,7 @@ public ProviderVolume create(ProviderAdapterContext context, ProviderAdapterData request, new TypeReference>() { }); - return (ProviderVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** @@ -140,22 +144,37 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d String volumeName = normalizeName(pod, dataObject.getExternalName()); try { FlashArrayList list = null; - FlashArrayHost host = getHost(hostname); - if (host != null) { - list = POST("/connections?host_names=" + host.getName() + "&volume_names=" + volumeName, null, + if (AddressType.NVMETCP.equals(volumeAddressType) && hostgroup != null) { + // NVMe-TCP pod volumes are connected at the host-group level so the + // array assigns a consistent NSID visible to every member host. + list = POST("/connections?host_group_names=" + hostgroup + "&volume_names=" + volumeName, null, new TypeReference>() { }); + } else { + FlashArrayHost host = getHost(hostname); + if (host != null) { + list = POST("/connections?host_names=" + host.getName() + "&volume_names=" + volumeName, null, + new TypeReference>() { + }); + } } if (list == null || list.getItems() == null || list.getItems().size() == 0) { - throw new RuntimeException("Volume attach did not return lun information"); + throw new RuntimeException("Volume attach did not return connection information " + + "(expected lun for Fibre Channel or nsid for NVMe-TCP)"); } FlashArrayConnection connection = (FlashArrayConnection) this.getFlashArrayItem(list); + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // The FlashArray REST API does not return nsid in the connections + // payload for NVMe-TCP. The namespace is identified on the host by + // EUI-128 (see FlashArrayVolume.getAddress()); the value returned + // here is stored by the driver only for informational purposes. + return connection.getNsid() != null ? "" + connection.getNsid() : "1"; + } if (connection.getLun() == null) { throw new RuntimeException("Volume attach missing lun field"); } - return "" + connection.getLun(); } catch (Throwable e) { @@ -167,15 +186,32 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d }); if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null && conn.getHost().getName() != null && + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // Prefer a hostgroup-scoped match when a hostgroup is configured + // on the pool; otherwise fall through to matching the connection + // by host like the Fibre Channel branch below. Covers both + // transport=nvme-tcp deployments with and without hostgroup=. + if (hostgroup != null && conn.getHostGroup() != null + && conn.getHostGroup().getName() != null + && conn.getHostGroup().getName().equals(hostgroup)) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } + if (conn.getHost() != null && conn.getHost().getName() != null + && (conn.getHost().getName().equals(hostname) + || (hostname.indexOf('.') > 0 + && conn.getHost().getName() + .equals(hostname.substring(0, hostname.indexOf('.')))))) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } + } else if (conn.getHost() != null && conn.getHost().getName() != null && (conn.getHost().getName().equals(hostname) || conn.getHost().getName().equals(hostname.substring(0, hostname.indexOf('.')))) && conn.getLun() != null) { return "" + conn.getLun(); } } - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection identifier (lun/nsid) not found in existing connection"); } else { - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection is not found in existing connection list"); } } else { throw e; @@ -238,7 +274,7 @@ public ProviderVolume getVolume(ProviderAdapterContext context, ProviderAdapterD } FlashArrayVolume volume = null; try { - volume = getVolume(externalName); + volume = withAddressType(getVolume(externalName)); // if we didn't get an address back its likely an empty object if (volume != null && volume.getAddress() == null) { return null; @@ -260,14 +296,24 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address throw new RuntimeException("Invalid search criteria provided for getVolumeByAddress"); } - // only support WWN type addresses at this time. - if (!ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + String serial; + if (ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + // Strip the NAA prefix (1 char) + Pure OUI to recover the volume serial. + serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); + } else if (ProviderVolume.AddressType.NVMETCP.equals(addressType)) { + // Reverse the EUI-128 layout: serial = eui[2:16] + eui[22:32], after + // stripping the optional "eui." prefix that appears in udev paths. + String eui = address.startsWith("eui.") ? address.substring(4) : address; + if (eui == null || eui.length() != 32) { + throw new RuntimeException("Invalid NVMe-TCP EUI-128 address [" + + address + "]: expected 32 hex characters, got " + + (eui == null ? "null" : String.valueOf(eui.length()))); + } + serial = (eui.substring(2, 16) + eui.substring(22)).toUpperCase(); + } else { throw new RuntimeException( "Invalid volume address type [" + addressType + "] requested for volume search"); } - - // convert WWN to serial to search on. strip out WWN type # + Flash OUI value - String serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); String query = "serial='" + serial + "'"; FlashArrayVolume volume = null; @@ -281,7 +327,7 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address return null; } - volume = (FlashArrayVolume) this.getFlashArrayItem(list); + volume = withAddressType((FlashArrayVolume) this.getFlashArrayItem(list)); if (volume != null && volume.getAddress() == null) { return null; } @@ -318,8 +364,11 @@ public ProviderSnapshot snapshot(ProviderAdapterContext context, ProviderAdapter "/volume-snapshots?source_names=" + sourceDataObject.getExternalName(), null, new TypeReference>() { }); - - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools. Without this, the adaptive + // driver persists the snapshot with an FC-style WWN and subsequent + // revert/list operations cannot locate the namespace. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** @@ -362,7 +411,12 @@ public ProviderSnapshot getSnapshot(ProviderAdapterContext context, ProviderAdap "/volume-snapshots?names=" + dataObject.getExternalName(), new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools instead of the FIBERWWN + // default. Without this, the adaptive driver persists the snapshot + // path with an FC-style WWN and revert/list fails to locate the + // namespace on the host. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } @Override @@ -599,6 +653,13 @@ private void login() { } } + String transport = connectionDetails.get(FlashArrayAdapter.TRANSPORT); + if (transport == null) { + transport = queryParms.get(FlashArrayAdapter.TRANSPORT); + } + volumeAddressType = TRANSPORT_NVME_TCP.equalsIgnoreCase(transport) + ? AddressType.NVMETCP : AddressType.FIBERWWN; + // retrieve for legacy purposes. if set, we'll remove any connections to hostgroup we find and use the host hostgroup = connectionDetails.get(FlashArrayAdapter.HOSTGROUP); if (hostgroup == null) { @@ -778,7 +839,14 @@ private FlashArrayVolume getSnapshot(String snapshotName) { FlashArrayList list = GET("/volume-snapshots?names=" + snapshotName, new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); + } + + private FlashArrayVolume withAddressType(FlashArrayVolume vol) { + if (vol != null) { + vol.setAddressType(volumeAddressType); + } + return vol; } private Object getFlashArrayItem(FlashArrayList list) { @@ -1087,7 +1155,15 @@ public Map getConnectionIdMap(ProviderAdapterDataObject dataIn) if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null) { + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // Host-group-scoped NVMe connections come back as one + // entry per host in the group; key on the host name so + // connid. is matched in parseAndValidatePath. + if (conn.getHost() != null && conn.getHost().getName() != null) { + String id = conn.getNsid() != null ? "" + conn.getNsid() : "1"; + map.put(conn.getHost().getName(), id); + } + } else if (conn.getHost() != null) { map.put(conn.getHost().getName(), "" + conn.getLun()); } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java index 76cec9f70c4b..b115035fda21 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java @@ -31,6 +31,8 @@ public class FlashArrayConnection { private FlashArrayVolume volume; @JsonProperty("lun") private Integer lun; + @JsonProperty("nsid") + private Integer nsid; public FlashArrayConnectionHostgroup getHostGroup() { return hostGroup; @@ -64,5 +66,12 @@ public void setLun(Integer lun) { this.lun = lun; } + public Integer getNsid() { + return nsid; + } + + public void setNsid(Integer nsid) { + this.nsid = nsid; + } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java index a3201a753a75..8343283d9952 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java @@ -27,6 +27,10 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class FlashArrayVolume implements ProviderSnapshot { public static final String PURE_OUI = "24a9370"; + // The 3-byte OUI as it appears inside an NVMe EUI-128 (no trailing nibble). + // FC WWNs use a 7-hex-digit Pure OUI; NVMe NGUIDs embed the same vendor + // prefix in its raw 6-hex-digit form. + public static final String PURE_OUI_EUI = "24a937"; @JsonProperty("destroyed") private Boolean destroyed; @@ -107,6 +111,19 @@ public AddressType getAddressType() { @JsonIgnore public String getAddress() { if (serial == null) return null; + if (AddressType.NVMETCP.equals(addressType)) { + // EUI-128 layout for FlashArray NVMe namespaces: + // 00 + serial[0:14] + + serial[14:24] + // This is the value the Linux kernel exposes as + // /dev/disk/by-id/nvme-eui. + if (serial.length() < 24) { + throw new RuntimeException("FlashArray serial [" + serial + + "] is too short to build an NVMe EUI-128 address " + + "(expected at least 24 hex characters, got " + + serial.length() + ")"); + } + return ("00" + serial.substring(0, 14) + PURE_OUI_EUI + serial.substring(14)).toLowerCase(); + } return ("6" + PURE_OUI + serial).toLowerCase(); } @Override