diff --git a/conf/db/upgrade/V5.5.28__schema.sql b/conf/db/upgrade/V5.5.28__schema.sql index e69de29bb2d..8ab469b732b 100644 --- a/conf/db/upgrade/V5.5.28__schema.sql +++ b/conf/db/upgrade/V5.5.28__schema.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS `zstack`.`ExternalPrimaryStorageHostProtocolRefVO` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `hostUuid` varchar(32) NOT NULL, + `primaryStorageUuid` varchar(32) NOT NULL, + `protocol` varchar(32) NOT NULL, + `status` varchar(32) NOT NULL, + `createDate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `lastOpDate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `ukExternalPrimaryStorageHostProtocolRefVO` (`primaryStorageUuid`, `hostUuid`, `protocol`), + CONSTRAINT `fkExternalPrimaryStorageHostProtocolRefVOHostEO` FOREIGN KEY (`hostUuid`) REFERENCES `zstack`.`HostEO` (`uuid`) ON DELETE CASCADE, + CONSTRAINT `fkExternalPrimaryStorageHostProtocolRefVOPrimaryStorageEO` FOREIGN KEY (`primaryStorageUuid`) REFERENCES `zstack`.`PrimaryStorageEO` (`uuid`) ON DELETE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET = utf8; + +CALL DROP_COLUMN('ExternalPrimaryStorageHostRefVO', 'protocol'); diff --git a/conf/persistence.xml b/conf/persistence.xml index b66d6319ff7..c1d7eea7fa7 100755 --- a/conf/persistence.xml +++ b/conf/persistence.xml @@ -142,6 +142,7 @@ org.zstack.header.storage.addon.primary.ExternalPrimaryStorageSpaceVO org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostRefVO + org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO org.zstack.storage.ceph.backup.CephBackupStorageVO org.zstack.storage.ceph.backup.CephBackupStorageMonVO org.zstack.storage.ceph.primary.CephPrimaryStorageMonVO diff --git a/conf/serviceConfig/externalPrimaryStorage.xml b/conf/serviceConfig/externalPrimaryStorage.xml index e076cc55bc1..f2767b1b7f7 100644 --- a/conf/serviceConfig/externalPrimaryStorage.xml +++ b/conf/serviceConfig/externalPrimaryStorage.xml @@ -13,4 +13,8 @@ org.zstack.header.storage.addon.primary.APIDiscoverExternalPrimaryStorageMsg externalPrimaryStorage> + + org.zstack.header.storage.addon.primary.APIQueryExternalPrimaryStorageHostProtocolRefMsg + query + diff --git a/conf/serviceConfig/volume.xml b/conf/serviceConfig/volume.xml index bd0a010273f..99044c2fdc8 100755 --- a/conf/serviceConfig/volume.xml +++ b/conf/serviceConfig/volume.xml @@ -20,6 +20,10 @@ org.zstack.header.volume.APIChangeVolumeStateMsg + + org.zstack.header.storage.primary.APIChangeVolumeProtocolMsg + + org.zstack.header.volume.APICreateVolumeSnapshotMsg diff --git a/docs/zbs-vhost/30-NEXT-SESSION.md b/docs/zbs-vhost/30-NEXT-SESSION.md new file mode 100644 index 00000000000..1be7ce183fd --- /dev/null +++ b/docs/zbs-vhost/30-NEXT-SESSION.md @@ -0,0 +1,210 @@ +# ZBS vhost — Next Session Handoff + +## ⏭️ 下个 session 从这里开始(2026-06-11 更新) + +**数据盘 vhost 真·可用性已证明 ✅**(见下「数据盘 vhost guest 级读写验证」)。剩余工作按优先级: + +> **下个 session 3 大设计待办(同一 framework,一起设计;细节见各自正文/决策记录)** +> 1. ✅ **per-protocol HA slot/grain 表形态 = 路 B(已实现,2026-06-15)** —— 锁路 B(保留继承 + 独立明细表)。决定性依据:`ExternalHostIdGetter` 槽位分配的「一行=一槽」不变量(`steppingAllocate`/`randomAllocateHostId` count 行数判满),路 A 多行会让 hostId 重复→`used.size()` 虚高→槽位撞号→HA fencing 腐败。已落地:删子类死 `protocol` 列(VO+metamodel+`V5.5.28` DROP_COLUMN);`checkHostAccessible` 协议化(volume.protocol!=null 读明细表,否则走折叠父表)。详见下「## 决策记录」。全量 premium build 绿。 +> 2. ✅ **systemTag 传 volume protocol(已实现,2026-06-15,user 指正改用 EphemeralPatternSystemTag)**:Cloud 建 VM **不填 DiskAO**,protocol 走 `VolumeSystemTags.VOLUME_PROTOCOL`。**类型 = `EphemeralPatternSystemTag`**(`extends PatternedSystemTag` → 带 `{protocol}` token **又** ephemeral;wire = `ephemeral::volumeProtocol::{protocol}`)。「临时」语义由框架免费提供:`createNonInherentSystemTags`(createVolume 已调)自动跳过 ephemeral tag 不落库,**无需手动 strip**。`VolumeManagerImpl.createVolume` 用 `getSystemTag(::isMatch)`+`getTokenByTag` 读进 `VolumeVO.protocol`(仿同包 `REQUIRED_INSTALL_URL`)。多数据盘消歧复用既有 `dataVolumeSystemTagsOnIndex["N"]`(positional,零新 schema);enum 校验在 `VolumeApiInterceptor.validate(APICreateVmInstanceMsg)` early-return 前。⚠️ 前端必须发 `ephemeral::` 前缀。全量 premium build 绿;IT 待重跑(改类型后)。 +> 3. **host vhost target 部署时机 = `ExternalPrimaryStorageHostProtocolRefVO` 进 connecting 时(2026-06-15 user 精确化)**:per-protocol ref 行建立连接(AddStorageProtocol(Vhost)/PS attach cluster 触发)→ ensure_target on that host;失败暴露运维操作面;现 VM-start 懒加载降为幂等兜底。**路 B 已锁 → 落点 = 明细表行 connecting**(非去继承行)。⏳ 本 session 未做。 + +1. **MR review/合并**:zstack **!10172** = 单 commit `df19328484`(vhost lazy deploy + volume protocol create/change,volume-msg+PUT,coderabbit 两条已落地+已 resolve);utility **!7256** = hugepage auto-size + lazy deploy 加固 `6c523c4cc`(coderabbit 5 条全修:shlex.quote 注入 / pull 失败落 tar+url fallback / target_healthy 三态(container_exists) / RLock 串行化 / insecure-registry 改 TLS 错误才降级重试;单测 30/30,`test_zbs_vhost_deploy.py` 新增 13 个)。都 target 上游 `feature-zbs-vhost`。⚠️ env1 主机上跑的还是旧版 .py,要复验加固行为需重推 agent 代码。 +2. **收敛 reportNodeHealthy 健康语义**(allMatch → per-protocol,防 vhost target 死拖垮整机 CBD connected)。 +3. **per-protocol 主机连接 = HA slot/grain 设计**(详见下「## 决策记录:per-protocol HA slot/grain(2026-06-15)」)。`ExternalPrimaryStorageHostRefVO.protocol` 死字段结论=**删**;per-protocol 状态去向(独立明细表 vs 切断继承让老表多行)**待最终拍板**。部署时机见顶部「3 大待办」第 3 条。与第 2 条 reportNodeHealthy per-protocol 同一 framework 改动。 +3b. **changeVolumeProtocol 防护(升级为内容探测 gate)**:①根盘:已装 OS 512e↔4Kn 双向切换都变砖 → 拒或强警告。②数据盘 CBD→Vhost:**老版本未发 physical_block_size,存量卷 fs 多为 512 元数据(xfs sectsz=512/小 ext4 1K block/512-LBA 分区表),mkfs 时刻冻结、升级不改,切 Vhost 必砖且无救援**(vhost block size 来自 SPDK blkcfg 不可 override)→ agent 切换前读卷头探测:xfs sb@0 的 sb_sectsize / ext4 sb@1024 的 s_log_block_size / MBR@510 / GPT@512,512-flavor 即拒。存量卷出路:倒数据重格式化 / 等 ZBS 512e bdev / 留 CBD。 +4. **清理测试残留**:vhost-livetest2-5(livetest5 已 Stopped,root 仍 Vhost)、`cbd-vhost-dvtest`(e21459f2,验证 VM,Running@190.207,挂 vhost-dvtest + cbd2vhost-test(70afb426,Vhost) + vhost2cbd-test(11c3f520,CBD))、vhost-switch-test 卷、env1 上的 rework jar/GlobalProperty(备份 `/root/vhost-rework-backup-*`,serviceConfig API 注册已挪到 volume.xml)。 +5. live-migration 目标机 `createWithFlags(0)` 不走 `Vm.start()` hook → hugepage 不自动扩(vhost VM 本就 offline-only,暂不修)。 +6. **ISO 离线交付 SPDK 镜像(方案已定,待打包+小 Java 改动)**:通道半现成——`Zbs.vhost.targetImageTar/Url` GlobalProperty(ZbsGlobalProperty.java:36/38)+ cmd 透传(fillVhostTargetParams:201-202)+ agent load_image tar/url 兜底都在,**但默认值全空 = 开箱死路**:默认 `targetImage=zbs-vhost:latest` 无 registry 前缀(pull 分支跳过)+ tar/url 空 → load_image 直接抛 absent;env1/env2 能跑全靠手工配 registry ref。落地三件事:① 构建期 `docker save` tar.gz 进 MN RPM(落 webapps 静态路径,8080 天然 served);② **fillVhostTargetParams 运行时推导 url**(property 空且本地交付 tar 存在 → 填 `http://{当前MN IP}:8080/zstack/zbs-vhost/`;注解 defaultValue 是编译期常量填不了 IP,安装期写死又怕 MN IP 变/多 MN);③ `targetImage` 默认换成带 tag 的无-registry ref。升级=换 tar+改 tag。小清理:download_image 用完不删 tmp tar。**打包落点已定(2026-06-12)**:进 **bin**(MN 安装/升级包,ISO 装机走 bin 自动继承;只进 ISO 会导致升级环境镜像版本与 agent 代码漂移)。bin 内与 all_in_one.tgz 平级(不塞 war),install/upgrade 脚本拷到 `webapps/zbs-vhost/` **独立目录**(不放 webapps/zstack 里,避免升级重铺 war 误删;URL=`http://{MN IP}:8080/zbs-vhost/`)。文件名带 tag,fillVhostTargetParams 扫目录取名推导 url,文件不存在回退现状(registry/报错)。体积实测(2026-06-12 @246.239):运行态 247MB / save tar 241MB / **save+gzip 83MB**(bin 实际增重),主 bin 直推无压力;体积不可接受时备选:拆独立 `zstack-zbs-vhost-image` 附加包进 ISO repo。调试走 /zstack-mn-bin-debug 解包重打包流程。下载链路索引:activateVhostVolume(ZbsStorageController.java:160)→fillVhostTargetParams(:196)→VHOST_ACTIVATE_PATH(:170)→zbs_storage_plugin.py:125→zbs_vhost_target.py:248 ensure_target→:198 load_image(image_present 短路=只第一次下)→**:99 docker pull 即下载本体**。 + +**env1 现状**:MN 跑 rework jar;`vhost-livetest5`(336967d0) Stopped;vhost 数据盘 `vhost-dvtest`(f510f28f) 已挂到 `cbd-vhost-dvtest`(e21459f2) 上且验证通过。 + +--- + +## 决策记录:per-protocol HA slot/grain(2026-06-15) + +**触发**:user 质疑「老表 `ExternalPrimaryStorageHostRefVO` 没新增字段就没必要」+「分协议展示 ps-host 连通有没有别的办法弥补,之前设计有问题」。一路掘到 HA fencer 形态。 + +### 掘出来的硬事实(全部 code-verified,不是推测) +- **`hostId` 是 HA 心跳槽位号**,不是身份 ID。agent 端 `ha_plugin.py:1482` `offset = host_id × heartbeat_required_space` —— 每 host 按 hostId 算偏移往共享心跳卷写时间戳,别人读各自槽判生死(sanlock 风格 lease slot)。 +- **ZBS 心跳卷 per-pool 且恒走 CBD**(`ZbsStorageController.activateHeartbeatVolume`:`CbdHeartbeatVolumeTO`,key=poolName)。**`KvmVhostNodeServer` 没有 self-fencer、不碰 hostId** → vhost 通路当前不参与 HA 心跳。 +- **缺口**:vhost 容器死 → CBD 心跳照跳 → host 不自隔离 → vhost VM 卡 IO 且无 HA 接管。单一 CBD 心跳对 vhost 数据面是瞎的。 +- **`getNodeSvc(psUuid)` 按 PS 取,不分协议**;一个 PS 一个 node service。 + +### slot 来源 = 两种策略,grain 跟来源走(核心顿悟) +| 策略 | 谁给号 | 天然 grain | 唯一性 | +|---|---|---|---| +| 存储派生(expon `getHostId()=uss.getServerNo()`,按协议查网关) | 存储侧网关 | per-(host,protocol) | 存储侧保证;**hazard**:两 host 同协议网关落同一 server 会撞号 | +| 自分配(ZBS `ExternalHostIdGetter` 1..999 池) | 我们自己叠 | 我们说了算 | 抽象池天然防撞 | + +**user 拍板**:自分配时定义成**跨协议一样**(给定 (ps,host) 一个号,所有协议心跳介质复用它)。号是我们发的就有这个自由;expon 的 per-protocol 是因为号来自 per-protocol 网关、被动接受。→ ZBS 1..999 池看着 per-host 纯属单协议偶然(只有 CBD 一条路参与 HA,两种 grain 数值重合)。 + +### 三层分解(正确终态) +- **连通性 status** → per-(ps,host,protocol)。 +- **心跳 slot/hostId** → 自分配统一放 per-(ps,host);存储派生不存、现查。 +- **父 `PrimaryStorageHostRefVO.status`** → 退化成折叠 rollup(折叠策略 any/all 待 checkHostAccessible 协议化后再定,届时不再 load-bearing)。 +- **放置门禁 `VolumeApiInterceptor.checkHostAccessible`(:457)** → 必须协议化:vhost 卷查 vhost 路状态,不是折叠值(否则 vhost 容器死、CBD 活=折叠 Connected → 把 vhost VM 放上去卡死)。 +- **roadmap**:加 vhost 独立心跳介质 + self-fencer + vhost-dir covering,复用 hostId。**不变量:一协议一心跳介质**(否则同槽号互相覆盖)。 + +### 表形态分歧(⏳ 待 user 最终拍板) +status 的真 grain = per-(ps,host,protocol),要落 DB。两条路: + +**路 A — 切断继承(user 倾向)**:`ExternalPrimaryStorageHostRefVO` 不再 extends `PrimaryStorageHostRefVO`,自己独立 → 天然能 per-(ps,host,protocol) 多行,复活那根死 `protocol` 列当判别列。 +- 代价:外部存储**离开多态家族**,下列 **4 个父表消费者**切断后看不到外部行,全要加「外部走新表 + 自算折叠 rollup」: + - `VolumeApiInterceptor:457` checkHostAccessible(放置门禁) + - `KVMHost:5417` inaccessiblePsCount(**host HA**:漏改 → 外部存储失联不计入整机隔离,最危险) + - `PrimaryStorageManagerImpl:1191`、`PrimaryStorageBase:543` +- 外部状态写路从「写父表」重定向到「写新表」;要写迁移(父+子 现存行搬进独立表)。 + +**路 B — 保留继承 + 独立明细表(assistant 倾向,= 已落地的现状增强)**: +- 父 `PrimaryStorageHostRefVO` 行**保留**当外部存储的**折叠 rollup**(per-(ps,host) 一行)+ 自分配 slot(hostId) 的家。host-HA / 粗放置免费继续用,零改。 +- per-protocol status 放**独立非继承表 `ExternalPrimaryStorageHostProtocolRefVO`**(已建好/已部署/已测,就是它)。 +- 删老 JOINED 子类那根死 `protocol` 列(子类瘦成只剩 hostId)。 +- 只把 `checkHostAccessible` 协议化(读明细表)。 + +**核心分歧点**:折叠的「host↔PS 连通」事实 **host-HA 真需要**。路 A 把它踢出父家族后,那个 rollup 你最后还得自己造出来喂 host-HA;路 B 让父行天然当 rollup。assistant 据此倾向 B(折叠行非垃圾,是 host-HA 合法消费的 rollup)。user 倾向 A(单张表更干净)。 + +**✅ 最终拍板:路 B(2026-06-15 实现)**。决定性新事实(不在上面论证里):`hostId` 槽位分配器的「一行=一槽」不变量——`ExternalHostIdGetter.steppingAllocate/randomAllocateHostId` 全靠 `select(hostId).listValues().size() == 行数` 判池满。路 A 让老表 per-(ps,host,protocol) 多行 → 同 hostId 在 N 个协议行里重复 → `used.size()` 虚高 → 槽位分配错乱 → 心跳 offset 撞号(`ha_plugin.py:1482 offset=host_id×space`)= HA fencing 数据腐败。且即便纯 A,hostId 仍要自己的 per-(ps,host) 家 → 还是两张表。路 B 用已落地明细表 + 父行天然当 rollup,14+ 父表消费者零改、无迁移。落地:删子类死 `protocol` 列(VO+metamodel+`V5.5.28 DROP_COLUMN`);`checkHostAccessible` 协议化(`volume.protocol!=null` → 读明细表,否则走折叠父行不变)。 + +> 注:item 2(reportNodeHealthy per-protocol)+ item 3(本决策)+ vhost self-fencer 是同一个 framework 改动,一起设计。当前已落地的「全家桶」是路 B 的雏形(明细表只装 status、未接 checkHostAccessible 协议化、未接 vhost fencer)。 + +### systemTag 传 volume protocol(2026-06-15 实现,路 B 配套) +- Cloud 建 VM 不填 DiskAO → 卷 protocol 只能走 systemTag。`VolumeSystemTags.VOLUME_PROTOCOL` = **`EphemeralPatternSystemTag`**(wire 形 `ephemeral::volumeProtocol::{protocol}`)。**关键纠正(user 指出)**:`EphemeralPatternSystemTag extends PatternedSystemTag`,**既带 `{protocol}` token 又是 ephemeral**——之前「EphemeralSystemTag 不能带 token」判断错了,错用了裸 PatternedSystemTag + 手动 strip。同包先例 `ExternalPrimaryStorageSystemTags.REQUIRED_INSTALL_URL`。详见 memory [[reference_ephemeral_pattern_systemtag]]。 +- **消费点 = `VolumeManagerImpl.createVolume(CreateVolumeMsg)`**(所有 VM-create 卷的汇聚点,VmAllocateVolumeFlow 把 tag 折进每卷 CreateVolumeMsg.systemTags):gate `vo.getProtocol()==null`(显式 msg.protocol 优先)→ `getSystemTag(VOLUME_PROTOCOL::isMatch)` 读 token + setProtocol。**不手动 strip**——`createNonInherentSystemTags`(createVolume:634 已调)对 ephemeral tag 自动跳过落库(TagManagerImpl:366),框架天然实现「落地即不常驻」。 +- **多盘消歧 = 复用现成 `dataVolumeSystemTagsOnIndex["N"]`**(VmAllocateVolumeFlow:72-78 按位置 setTags 到对应 VolumeSpec),裸 tag 进 `dataVolumeSystemTags` 则广播全数据盘。零新 tag schema。 +- **enum 守门 = `VolumeApiInterceptor.validate(APICreateVmInstanceMsg)` 前置**(在 getDiskAOs() 早返回之前,读 dataVolumeSystemTags + OnIndex):抄 validateVolumeProtocol:497 的 enum-known 检查 argerr 拒未知协议。PS-exposes-protocol 在 API 期查不了(PS 流程中段才分配),enum 是 fail-fast 层。 + +### ⚠️ 发现的潜在 product bug(本次未修,out of scope) +`VolumeBase.expunge`(:1177)只要 `primaryStorageUuid != null` 就跑 delete-on-PS flow,**无 NotInstantiated/installPath 守门**(对比 `delete` 用 `allowEmptyFlow()`)。外部 PS 上一个 NotInstantiated 卷(installPath=null)+ protocol≠null 被 expunge → `ExternalPrimaryStorage.deactivateAndDeleteVolume(null, protocol)` 跳过 null-guard → `node.getActiveClients(null, protocol)` NPE。正常流程卷在 expunge 前已 instantiate(有真 installPath),故少见;但「建数据卷指定 protocol→从不 attach→删→expunge」路径理论可触发。修法:expunge 的 delete-on-PS flow 加 `skip(){ return self.getInstallPath()==null }`。 + +--- + +## 已完成 / 已验证 + +### per-protocol 主机连通性全家桶(2026-06-12)✅ 实现+IT+双环境真机 +设计:父表 UNIQUE(ps,host) 索引(V4.3.12)封死同表多行 → **新表 `ExternalPrimaryStorageHostProtocolRefVO`**(ps,host,protocol,status,UNIQUE 三键,FK cascade),14 个 legacy 读点零改动。host-level 行语义改为**默认协议健康**。 +- **健康记录(按用户要求收敛为最小增量)**:KvmFactory.checkHostStatus **原折叠逻辑一行不动**(allMatch→整机行为与历史一致),只在 success 里加 2 行 forEach 把 NodeHealthy 每协议 fire-and-forget 发 `UpdatePrimaryStorageHostStatusMsg(+protocol)`;External handler protocol!=null 时 upsert 新表后直接 reply(不碰 legacy 行)。per-protocol 行=明细可观测层,不改变 HA/调度语义。 +- **AddStorageProtocol 触发主机准备**:`PrimaryStorageControllerSvc.onProtocolAdded` default 钩子;ZbsStorageController 实现 Vhost→对 Connected hosts 发 `VHOST_TARGET_ENSURE`,结果 fire-and-forget 写协议行(防 PS 队列死锁);失败不阻塞 API(ping 自愈)。 +- **查询 API**:`APIQueryExternalPrimaryStorageHostProtocolRefMsg`(GET /v1/external-primary-storage/host-protocol-refs,AutoQuery)+ SDK/ApiHelper regen。 +- **systag**:`volumeProtocol::{protocol}` 当时只留定义(reserved)不接入(**已于 2026-06-15 接入**,改 `EphemeralPatternSystemTag`,见顶部待办 2);DiskOffering.protocol 用户砍掉。 +- **IT**:ZbsVhostVolumeCase 新增 testAddProtocolPreparesHostsAndRecordsProtocolRefs(删协议行→API 重加→ensure 模拟器断言→协议行→查询 API→ping 驱动 Vhost 行独立翻转→自愈),全 case 过。顺手修了 VHOST_DEACTIVATE 无模拟器的历史 flake。 +- **真机(双环境 2026-06-12)**:①两环境各 6 行(3 host×CBD/Vhost)ping 自动建 Connected;②REST 查询 200;③**docker stop zbs-vhost → Vhost 行 Disconnected、CBD 行 Connected**(按协议明细精确定位坏的数据面;host-level 行保持折叠语义照旧判挂——该真机验证早于"最小增量"收敛,收敛后 host-level 行为=历史版);④删协议行→REST AddProtocol(Vhost)→246.239 容器重建、**248.246 全新自动部署**(pull+容器 26s Up)、242.235 无 docker 如实 Disconnected 且 API 成功。 +- **部署教训**:persistence.xml 是构建合并产物(premium 实体),不能整文件覆盖,要 sed 插行;apihelper 产物在 ~/ApiHelper.groovy 需手工回填 testlib。 +- 升级 SQL 落在 V5.5.28__schema.sql(原空文件);env1/env2 已手工建表。 + +### env2 (172.24.248.246) 干净环境 E2E + 加固代码实测(2026-06-11)✅ +加固版(`6c523c4cc`)在第二套环境从零跑通全链路,5 条 review 修复中 3 条拿到 live 证据: +- **部署**:env1 MN 4 jar(header/storage/zbs/sdk,md5 比对一致)+ serviceConfig 2 xml → env2 `/usr/local/zstacktest`(备份 `/root/vhost-rework-backup-env2/`);properties 加 `Zbs.vhost.targetImage`;DB 插 PS `d402277a` 的 Vhost outputProtocol 行(列名是 `outputProtocol` 不是 protocol);MN 重启 OK。agent 4 .py(**加固版** target/rpc/storage/vm_plugin)→ host 246.239 py3.11 venv(备份 `/root/vhost-py-backup/`),ansible reconnect 没冲掉 .py(md5 复验)。 +- **clean-slate 起点实证**:0 镜像/0 容器/daemon.json 无 insecure-registries/大页 0。 +- **E2E 主链**:CBD root(imagestore ttylinux qcow2 `146e9570`,实为 CentOS7)+ 显式 protocol=Vhost 数据卷 `d863c14f` → start 钉 246.239 → **21 秒 Running**。链路内自动完成:**先正常 docker pull→TLS 错→log WARNING "retrying registry as insecure"→写 daemon.json→重试拉下 247MB**(fix #5 条件降级 live 证据)→ 容器起 → 大页 0→335 → vhostuser socket 连。guest(netns ssh root/password)vdb 4Kn → mkfs.xfs/mount/16MB direct/remount md5 OK。 +- **reclaim E2E**:ZStack stop → 大页 335→185(容器 reserve 保留)。 +- **健康三态 live**(fix #3):running+sock→True;`docker stop`→**False**(旧代码这里误报 True);`docker rm -f`→True。 +- **幂等重部署**:容器被 rm 后再 start VM → **8 秒** Running,容器重建,日志 pull 总次数仍 1(image_present 短路,零重复拉取),大页回 335。 +- 未 live 覆盖:fix #2 fallback(registry 正常时走不到 tar/url,单测覆盖)、fix #4 锁(单流程无并发窗口,单测覆盖)。 +- **env2 残留**:VM `vhost-e2e-env2`(282235df, Stopped) + 卷 `vhost-e2e-dv`(d863c14f, Vhost);rework jar/xml/properties(回滚用 backup-env2);246.239 上 .py/镜像/容器/daemon.json registry 项。 +- **ansible kvmagent tar 已刷加固版(两环境,2026-06-11)**:env1+env2 MN 的 `WEB-INF/classes/ansible/kvm/kvmagent-5.5.0.tar.gz` 注入 6c523c4cc 的 4 .py(内层 md5=eb0b5b28 与本地一致;env1 原 tar 是 6/9 旧版注入、env2 原 stock)。原 tar 备份 `${TAR}.orig-backup`。Reconnect 物理机现在会铺**加固版**而非冲掉。SPDK 镜像无 tar 形态:只在 registry `172.26.208.212:5000`,`/var/lib/zstack/zbs-vhost-image.tar` 两环境都不存在(tar/url 只是 load_image 兜底参数,从未用过)。 + +### 数据盘 vhost guest 级读写验证(2026-06-11)✅ 闭环 +方法:绕开 4Kn-root 不引导 —— **CBD root + Vhost 数据盘**。 +- 流程:cold-stop `vhost-livetest5` → detach `vhost-dvtest`(f510f28f, Vhost) → 新建 `cbd-vhost-dvtest`(uuid `e21459f26c434159b17e61ce3f3b83fb`,CreateStopped,root CBD on ZBS PS,钉 190.207)→ attach → start → Running,IP `10.5.166.99`。 +- guest 实为 **CentOS 7**("ttylinux" 镜像 acf309cd = zstack-test-image,有 sshd root/`password`、mkfs.xfs 无 mkfs.ext4)。进 guest 走 host DHCP netns:`ip netns exec br_zsn0_1105_13a0eaea... ssh root@10.5.166.99`,比 send-key 省事得多。 +- **Linux 认盘 ✅**:lsblk 见 `vdb` 100M,4Kn(logical/physical block size 都 4096)。**上次 OVMF 不认 root-port 后数据盘 = 纯固件限制,Linux virtio-blk 正常枚举**,疑问解决。 +- **裸块 RW ✅**:`dd oflag=direct` 写 8MB → `iflag=direct` 读回,md5 一致。`mount` 报 "write-protected" 是无文件系统时的回退假报警,`/sys/block/vdb/ro`=0。 +- **文件系统 RW + 持久 ✅**:mkfs.xfs(sectsz=4096)→ mount(df 确认源=/dev/vdb)→ 32MB direct 写 + proof.txt → sync → umount → remount → md5 OK + 文件完整。 +- **SPDK 链路佐证 ✅**:容器内 `rpc.py -s /var/tmp/vhost-sockets/vhost.sock bdev_get_iostat` → `zbs-bdev-b635bfd3...` written 50.7MB / read 75MB,与 guest I/O 量吻合,流量真走 qemu vhost-user → SPDK → ZBS。 +- ⚠️ 教训:第一轮 mkfs.ext4 不存在 + mount 失败,dd 全写进 root 盘还 md5 "OK"(假阳性);**fs 级验证必须 df 确认挂载源**。 + +### 协议双向切换闭环(2026-06-11)✅ CBD→Vhost 与 Vhost→CBD 都验过 +- **A** `cbd2vhost-test`(70afb426):CLI 默认创建(protocol=CBD)→ `PUT /zstack/v1/volumes/{uuid}/actions {"changeVolumeProtocol":{"protocol":"Vhost"}}` → DB Vhost → attach `cbd-vhost-dvtest` 起机 → host XML **vhostuser** socket `zbs-vhost-a58598eb...` → guest vdc:raw dd direct 写读 md5 一致 + mkfs.xfs/mount/16MB/umount/remount md5 OK。 +- **B** `vhost2cbd-test`(11c3f520):REST 创建显式 protocol=Vhost → changeVolumeProtocol CBD → DB CBD → attach 同 VM → host XML **network CBD** 盘 serial=11c3f520(无 vhost socket)→ guest vdd:raw + fs 持久全过。 +- 未挂载卷切协议不受 VM 状态限制(interceptor 停机校验只约束已挂卷),Ready 态直接切成功。 +- **guest 识盘方法**:vhost-user-blk 的 serial = SPDK bdev 名(`zbs-bdev-`,hash 与 per-vol socket 名一致);CBD 盘 serial = volume uuid 前 20 字符。`cat /sys/block/vd*/serial` 即可精确映射,不用猜设备序。 + +### 扇区语义 + 带数据跨扇区切换(2026-06-11 续)✅ +- 实测确认:CBD = **512e**(guest logical 512 / physical 4096,qemu 块层 RMW 桥),vhost = **4Kn**(4096/4096)。切协议不动字节内容,只改 guest 看到的 logical sector size。 +- **带数据双向切换实测过**:A(Vhost/4Kn 下写的 xfs+16MB 数据)切到 CBD/512e → mount + md5 OK;B(CBD/512e 下写的)切到 Vhost/4Kn → mount + md5 OK。前提=整盘 xfs sectsz=4096(mkfs.xfs 在 CBD 上因 physical=4096 hint 默认就选 4096,默认路径安全)。 +- **数据盘双向切换的边界**:带 MBR/GPT 分区表的盘(GPT header 在 LBA1,512↔4096 下字节偏移不同)切换后分区表解释错位,**双向都坏**(推理确定,未实测);显式 `mkfs.xfs -s size=512` 的 fs 切 4Kn 后 mount 拒(fs sectsz < device logical)。 +- **根盘双向都不能安全切(已装 OS)**:CBD→Vhost = 512-LBA 装的 GPT/ESP 在 4Kn 下找不到 + SeaBIOS 一律 fail(vhost-livetest5 卡 UEFI shell 实证);Vhost→CBD = 4Kn 装的 GPT 写在 byte 4096,512e 下 LBA1=byte 512 同样找不到。**不存在"根盘只能 vhost→cbd"的单向通道**。唯一正解 = ZBS 出 512e bdev 消除扇区差(长期方案,已记)。 +- 产品防护 follow-up:changeVolumeProtocol interceptor 当前不区分 root/data,根盘切换=自助翻车,考虑拒绝 root volume 或强警告。 + +### blockio logical 4K 解锁 Vhost→CBD 切换(2026-06-11 实测)✅ +用户提出:CBD 侧指定 block io 4K 应该就能切。**实测成立**(数据盘 MBR 级证明,190.207 手工 domain XML): +- 实验:B 卷在 Vhost/4Kn 下 fdisk 建 MBR(vdd1 起始 LBA 256=byte 1MiB)+ xfs + 16MB 数据 → 切 CBD 默认 512e → vdd1 被解释成 12.4M、起点 128KiB、blkid 无 fs、mount 失败(砖)→ 给该盘 XML 的 blockio 加 `logical_block_size='4096'`(ZStack 原生已发 `physical_block_size='4096'`,**只缺 logical**)→ virsh create → 分区恢复 99M、xfs 直挂、md5 OK。 +- 注意:该盘 XML 已有 ``,**插第二个 blockio 元素会被 libvirt 静默丢弃**,必须改写已有元素。 +- 含义:**Vhost→CBD 根盘切换可行**(4Kn 装的 OS 切 CBD 后加 logical 4K,UEFI 引导语义不变);且 CBD 4K 化后 CBD↔Vhost 双向都安全(两端同为 4Kn)。代价=该卷永失 SeaBIOS legacy 兼容(seabios blksize!=512 fail)。 +- 反向不可对称修复:vhost-user-blk 的 block size 来自 SPDK 后端 blkcfg(VHOST_USER_GET_CONFIG),qemu XML 无法 override → 512 装的 OS 切 Vhost 仍无解,只能等 ZBS 512e bdev。 +- 产品化落点:changeVolumeProtocol Vhost→CBD 时给卷打标(如 sectorSize=4096),KVM agent 生成 CBD 盘 XML 时按标发 logical 4096。未做,记 follow-up。 +- 残留:实验后已 virsh destroy + ZStack 标准重起(B 回到默认 512e,其 MBR 视图重新"砖",测试卷无害)。boot 级(OVMF 真引导 4Kn CBD 根盘)未直接测,置信高(OVMF 已证能引导 4Kn vhost,virtio config 语义同)。 + +### 单 vhost(更早的 session,已 done) +- Alpine 3.23 装到 ZBS vhost 4Kn 盘、UEFI+Q35 独立引导到 login(真机)。 +- 块大小根因锁定:bdev_zbs 暴露 4Kn → SeaBIOS(legacy) 起不来(seabios `virtio-blk.c:146 blksize!=512 goto fail`,Dell/MS/Intel 文献 + qemu 源码三方印证);cbd 能 legacy 是 qemu 块层做 512↔4096 RMW(`cbd.c:467 request_alignment=4096` + `io.c` pad),vhost 旁路块层故无此桥。正解 = ZBS 出 512e bdev(长期),当前用 UEFI+4K 镜像。 +- Groovy 全链路 case `ZbsVhostVolumeCase` 过;MR !10101(zstack)/!7208(utility) target 5.5.28。 + +### 本 session 增量(代码全部 compiled + Groovy 过 + pushed) +3 个功能: +1. **全链路自动部署(懒加载)** — `zbs_vhost_target.py`:首次 activate 时 `load_image` 主路径 **registry docker pull**(从 image ref 解析 registry,自动配 insecure-registry + SIGHUP reload + pull),tar/url 兜底;`compute_cores` 按主机 CPU 取高位核;`ensure_target` 懒起容器。 +2. **vhost health** — `reportNodeHealthy` 加 Vhost 分支(gate `supportsVhost()`)→ 新 `VHOST_TARGET_HEALTH_PATH` 端点(`target_healthy`:未部署=健康、部署但 sock 死=不健康)。 +3. **切换协议 API** — `APIChangeVolumeProtocolMsg`(离线、卷级、interceptor 校验:卷在 external PS、目标协议在 PS outputProtocols、VM 停机),handler 在 `ExternalPrimaryStorage.doChangeVolumeProtocol` 更新 `VolumeVO.protocol`。新 API 进了 `conf/serviceConfig/primaryStorage.xml` + SDK regen(`ChangeVolumeProtocolAction`)。 + +commit: zstack `757b06acb9`(+base `a9bc5f4d8a`)、utility `92e36805f`,已 push。 + +### 懒加载全链路真机已验证(env1,2026-06-09,host 190.207)✅ 本次完成 +切 `vhost-livetest5` root CBD→Vhost + UEFI/q35 tag,强制起在 **190.207**(242.132 无 docker,跳过)。一次跑通: +- ✅ **registry 自动 pull**:`load_image` 从 image ref 解析 registry → 自动写 `daemon.json` `insecure-registries:[172.26.208.212:5000]` + `systemctl reload docker` + `docker pull`,image `zbs-vhost-x86_64:fc5404056` 落地。 +- ✅ **容器自动起**:`zbs-vhost` 容器 running,control sock `/var/tmp/vhost-sockets/vhost.sock` ready。 +- ✅ **hugepage 自动分配**:plugin `ensure_hugepages` 配 256 页(compact_memory 后)。 +- ✅ **vhost 盘挂载**:domain XML `` target vda boot order 1;SPDK 建 per-vol sock `zbs-vhost-edc5e2d5...`;qemu cmdline `vhost-user-blk-pci ... bootindex=1`;`ss -x` 实证 qemu↔SPDK socket 已连。 +- ✅ **VM Running + VNC**:domid 3,VNC `:1`→5901 RFB 003.008 握手通。 + +#### ⚠️ 本次发现关键 gap:hugepage 只 size 了 target,没算 guest RAM +首次 start 失败:`qemu-kvm: unable to map backing store for guest RAM: Cannot allocate memory`。 +根因:vhost-user-blk 强制 guest RAM 走共享 hugepage(qemu `-object memory-backend-memfd hugetlb:true share:true prealloc:true`),即**每个 vhost VM 的 guest 内存也吃 hugepage**(300MB guest = 150×2MB 页)。但 `DEFAULT_HUGEPAGE_NR=256` 只够 SPDK target(~185 页),剩 71 < 150 → 分配失败。 +临时绕过:手动 `echo 768 > nr_hugepages`(host 24G)→ 583 free → start 成功。 +**待修(代码)**:`ensure_hugepages` 要 size = target_reserve + Σ(本机 vhost VM guest RAM),或 activate 时按需扩容;否则多/大 vhost VM 必撞墙。这是懒加载落地必修项,优先级高于下面的 framework 增强。 + +### 真机已验证(env1 172.24.191.9) +- 部署:4 jar(header/sdk/storage/zbs) + serviceConfig + GlobalProperty 到 `/usr/local/zstacktest/`,agent .py 到 3 台主机,MN 重启 OK(无 API 错误)。备份在 `/root/vhost-jar-backup`。 +- ✅ **vhost health 端点**:MN 调 `/zbs/primarystorage/vhost/target/health`,3 台主机回 `healthy:true`。 +- ✅ **切换协议 API**:REST `POST /zstack/v1/volumes/{uuid}/protocol {"params":{"protocol":"Vhost"}}` 把数据卷 CBD→Vhost,DB 确认(CLI 不认新命令,必须走 REST)。 +- ✅ **VM 能起**:CBD VM `vhost-livetest5`(uuid 336967d0, root 3905ed5e) Running。 + +### 创建时指定 volume protocol(2026-06-09)✅ 真机验证 +之前只有 changeVolumeProtocol(事后切,更新路径)。补齐 create 路径:建卷时直接指定协议,覆盖 PS defaultProtocol。 +- 改动(5 处):`APICreateDataVolumeMsg` 加 `@APIParam(required=false) protocol`;`CreateDataVolumeMsg` 加 protocol 字段;`VolumeManagerImpl.handle(APICreateDataVolumeMsg)` 透传 + `handle(CreateDataVolumeMsg)` `vo.setProtocol(msg.getProtocol())`;`VolumeApiInterceptor` 加 `validateVolumeProtocol`(protocol 非空→enum 合法 + 必须带 primaryStorageUuid + protocol ∈ PS outputProtocols,复用 change 路径同款 `PrimaryStorageOutputProtocolRefVO` 校验)。create 走 blank instantiate,`ExternalPrimaryStorage` 只在 protocol 空时填 default → 显式协议存活。 +- Groovy:`ZbsVhostVolumeCase.testCreateDataVolumeWithExplicitProtocol`(显式 CBD 覆盖 Vhost 默认 + NBD 未暴露被拒)。 +- **真机实证(env1 MN,部署 header+storage jar 重启)**:REST `POST /zstack/v1/volumes/data {protocol:Vhost}` → 卷 protocol=Vhost(覆盖 PS default=CBD);`protocol:NBD` → 拒 `does not expose output protocol[NBD]`;`protocol:Vhost` 无 PS → 拒 `primaryStorageUuid is required`。env1 PS `ZStone_ZBS_PS` outputProtocols={CBD,Vhost} 已 loaded(`PrimaryStorageOutputProtocolRefVO`)。 +- **SDK regen / 构建(隔离 .m2 重做后全绿)**:⚠️ 头一次构建没隔离 .m2,全局 `~/.m2` 被并发 build clobber(`utils-5.5.0.jar` 被覆盖致版本 skew)→ `sdk` profile 编 `premium/test-premium` 报 `FieldUtils/TypeUtils/CollectionDSL cannot be resolved`,**误判成 pre-existing breakage(错的)**。`runMavenProfile` 其实 auto-detect `zstack/.m2/repository`——`mkdir + cp -r .m2-baseline/repository` 后自动隔离。隔离重做:premium build SUCCESS → `sdk` regen SUCCESS(出 `CreateDataVolumeAction.protocol`)→ test 模块 compile SUCCESS(`ZbsVhostVolumeCase.testCreateDataVolumeWithExplicitProtocol` 的 `createDataVolume{protocol=}` DSL 解析通过、class 产出)。详见 memory [[feedback_m2_isolation]]。 +- 验证齐备:(1) REST 真机三连过(Vhost 建卷成功 + NBD/无PS 被拒);(2) 隔离构建 + SDK regen 全绿;(3) **Groovy `ZbsVhostVolumeCase` IT 跑通**(Tests run:1 Failures:0 Errors:0,74.6s;log 实证 explicit-nbd 被 `does not expose output protocol[NBD]` 拒)。SDK 生成只动 `sdk/CreateDataVolumeAction.java`。 +- **跑 case 才抓到的坑**:`runMavenProfile sdk` 只改 SDK 源码不重装 jar → `.m2` 里 `sdk-5.5.0.jar` 仍无 protocol 字段 → DSL `createDataVolume{protocol=}` 运行时 `MissingPropertyException`(编译期不报)。修:regen 后 `mvn -pl sdk install`。跑 IT case 用 `scripts/run-zstack-case.sh`(flock + stray-JVM 预检 + timeout kill 子树),见 [[feedback_it_case_serial]]。 + +### 踩坑(关键) +- VM 起不来根因(日志 `PrimaryStorageMainAllocatorFlow`):**镜像类型不匹配** —— Ceph-BS 上的镜像 `possiblePrimaryStorageTypes=["Ceph"]`,ZBS PS 是 **Addon** 类型 → 被剔除。**必须用 imagestore BS 上的镜像**(`imagestore1` 69665b28...),如 ttylinux qcow2 `acf309cd17284144a5a7d130767a6ef4`。Ceph-BS 镜像(如 ttylinux raw 5754344f)在 Addon PS 上用不了。 +- `ExternalPrimaryStorageHostRefVO.protocol` 列是**死字段**:从不 setProtocol、从不被 query、allocator 也不引用 → 协议级主机连接未实现。 +- AddStorageProtocol 不触发 host connect(host ref 靠 `ExternalPrimaryStorageKvmFactory.checkHostStatus`→reportNodeHealthy→UpdatePrimaryStorageHostStatusMsg 按 status 变化才建)。 +- `reportNodeHealthy` 消费方 `ExternalPrimaryStorageKvmFactory:240` 用 `allMatch(Ok)` 折叠协议 → 加了 Vhost 后,vhost target 死会把整机对该 PS 判 Disconnected(含 CBD)。隐患,待收敛。 +- commit-msg hook 3 警告即停:body 每行 ≤72 消 line-length 警告。worktree `.git` 是文件 → `zdev_git_commit` 直 commit 报 ENOTDIR,用 dry_run 取 Change-Id 再 `git commit -F`。 + +## 下一步要做 + +1. ~~**补完懒加载真机验证**~~ ✅ DONE(见上「懒加载全链路真机已验证」)。残留只剩 guest 不引导(ttylinux 512 on 4Kn,预期;要 4K-UEFI 镜像如 Alpine 才进 login)。 +1b. ~~**修 hugepage sizing**~~ ✅ DONE(2026-06-09,free-based 按需分配)。 + - 设计:放弃固定 256 / libvirt 汇总 / MN 传字段。用 **free-based 增量**——内核已报 `HugePages_Free`,"running guests + 容器" 隐含在 `total-free` 里。规则:起 VM 那刻 `need=ceil(vm_mem/2MB)`,`free 面向:前端 / UI 团队 +> 后端分支:`feature-zbs-vhost`(zstack !10172) +> 本期口径:**三个 API 均为"接口预留"**——契约(API 名 / REST 路径 / 参数 / 校验语义)已冻结可对接, +> 但产品本期不承诺全量后端能力(见文末「实现状态总表」)。 + +## 1. 背景 + +外接主存储(Addon / vendor 类型,如华瑞 ZStone)一套存储可以同时暴露多种数据面协议。 +例:一套华瑞可同时添加 **iSCSI** 和 **Vhost**。 + +- 协议的**默认实现由 ZStack 负责**:VM domain XML 的构建与管理(vhost-user / iSCSI 等盘的接线)全部在 ZStack 侧完成,vendor 只提供数据面。 +- 卷(Volume)粒度记录自己的协议;同一主存储上不同卷可用不同协议。 + +## 2. 协议枚举(`VolumeProtocol`) + +``` +NVMEoF | iSCSI | Vhost | CBD | NBD | RBD +``` + +- 字符串严格区分大小写,传值必须与枚举一致(是 `NVMEoF` 不是 `Nvme`;是 `Vhost` 不是 `vhost`)。 +- 一套 PS 实际支持哪些协议,**以该 PS inventory 的 `outputProtocols` 为准**(见 §3),前端下拉必须用它过滤,不要写死枚举全集。 + +## 3. 前端可查询的字段(现成,可直接用) + +### 3.1 主存储(`ExternalPrimaryStorageInventory`) + +| 字段 | 类型 | 说明 | +|---|---|---| +| `outputProtocols` | List\ | 该 PS 已添加的协议集合(驱动下拉/标签展示) | +| `defaultProtocol` | String | 不显式指定协议时新建卷的默认协议 | + +查询:`QueryPrimaryStorage`(GET `/v1/primary-storage`),Addon 类型 PS 返回上述字段。 + +### 3.2 卷(`VolumeInventory`) + +| 字段 | 类型 | 说明 | +|---|---|---| +| `protocol` | String | 卷当前协议(列表页建议加列展示) | + +### 3.3 按协议的主机连通性 — **已实现,可直接对接** + +每台主机对每个协议的数据面连通性记录在**独立新表** `ExternalPrimaryStorageHostProtocolRefVO`(host × PS × protocol 一行,protocol **非空**,无 NULL 语义),通过专属查询 API 暴露: + +**`APIQueryExternalPrimaryStorageHostProtocolRefMsg`** + +``` +GET /zstack/v1/external-primary-storage/host-protocol-refs?q=primaryStorageUuid=xxx +GET /zstack/v1/external-primary-storage/{primaryStorageUuid}/host-protocol-refs +``` + +返回 `inventories`(`ExternalPrimaryStorageHostProtocolRefInventory`): + +| 字段 | 类型 | 说明 | +|---|---|---| +| hostUuid | String | 主机 UUID(支持 expand:`host.name=xxx` 条件) | +| primaryStorageUuid | String | 主存储 UUID | +| protocol | String | 协议(CBD / Vhost / ...,每个协议独立一行) | +| status | String | `Connected` / `Connecting` / `Disconnected`,**按协议独立** | +| createDate / lastOpDate | Timestamp | lastOpDate = 状态最近变化时间 | + +SDK:`QueryExternalPrimaryStorageHostProtocolRefAction`。 + +**状态语义**: +1. 行的产生:主机连接/周期 ping 时按协议健康上报自动建行/刷新;`AddStorageProtocol` 成功准备主机后也会立即写入该协议的行。 +2. **整机(host-level)状态保持历史折叠语义**(所有上报协议的 AND,任一协议不健康整机置 Disconnected),HA/调度/挂盘判定行为与升级前完全一致;**本表提供按协议的独立明细**——如 Vhost target 挂掉时,本表能精确显示 Vhost 行 Disconnected 而 CBD 行 Connected,定位是哪条数据面坏了。 +3. 旧 `ExternalPrimaryStorageHostRefVO.protocol` 字段为历史遗留列(建表即有、无代码读写),按协议状态一律以本表为准。 + +**UI 建议**:PS 详情页「主机连通性」表格(列:主机 / 协议 / 状态 / 更新时间),按 hostUuid 分组多协议行展示。状态色:Connected 绿 / Connecting 黄 / Disconnected 红。 + +## 4. API 一:加载协议到主存储 + +**`APIAddStorageProtocolMsg`**(注意拼写:Protocol,产品侧笔记里 "APIAddStoragePotocol" 少了 r) + +``` +POST /zstack/v1/primary-storage/protocol +{ + "params": { + "uuid": "fbfc898799694dfa9336d09690bd7e41", // 主存储 UUID + "outputProtocol": "Vhost" // 要添加的协议 + } +} +``` + +| 参数 | 必填 | 例子 | 备注 | +|---|---|---|---| +| uuid | 是 | xxx | 主存储 UUID | +| outputProtocol | 是 | `Vhost` | §2 枚举值 | + +- 返回:`APIAddStorageProtocolEvent`(标准 async event,轮询 location)。 +- 效果:PS 的 `outputProtocols` 集合追加该协议。 +- 可重复调用添加多个协议(iSCSI + Vhost 共存)。 +- SDK:`AddStorageProtocolAction`。 +- **行为(已实现)**:添加协议后会对该 PS 所有已连接主机**触发按协议的主机准备**(Vhost → 自动部署 SPDK target),并把每台主机该协议的连通性写入 §3.3 的按协议状态表。主机准备失败不阻塞 API(协议已注册,下个 ping 周期自愈),UI 可在 §3.3 表格里观察每台主机的就绪状态。 + +## 5. API 二:指定协议创建盘 + +**`APICreateDataVolumeMsg`**(在现有创建数据卷 API 上新增可选参数) + +``` +POST /zstack/v1/volumes/data +{ + "params": { + "name": "dv-1", + "diskOfferingUuid": "...", + "primaryStorageUuid": "fbfc8987...", // 指定 protocol 时必填 + "protocol": "Vhost" // 可选;缺省走 PS defaultProtocol + } +} +``` + +| 参数 | 必填 | 备注 | +|---|---|---| +| protocol | 否 | §2 枚举;**传了就必须同时传 `primaryStorageUuid`**,且协议必须 ∈ 该 PS `outputProtocols` | + +- SDK:`CreateDataVolumeAction` 已带 `protocol` 字段。 +- **协议来源优先级(设计约定,前端按此渲染表单逻辑)**: + 1. API `protocol` 参数(最高) + 2. systemTag `ephemeral::volumeProtocol::{protocol}`(**已实现,仅 VM 创建链路**,见 §5.1) + 3. `DiskOfferingVO.protocol`(**已取消**,产品决定不做) + 4. PS `defaultProtocol`(兜底) + + 本独立建卷 API(`APICreateDataVolumeMsg`)实际生效的是 1 和 4——它**不消费** systemTag(直接用 `protocol` 字段)。优先级 2 只在 VM 创建链路生效。 + +### 5.1 VM 创建时按 systemTag 指定卷协议 — **已实现** + +Cloud 建 VM 走 `APICreateVmInstanceMsg`,数据盘由 `dataDiskOfferingUuids` 描述,**没有 per-盘的 protocol 字段**。要在建 VM 时给数据盘指定协议,只能通过 systemTag: + +`APICreateVmInstanceMsg` 的两个现成 systemTag 通道(已有字段,无需新增): + +| 字段 | 类型 | 作用域 | 用法 | +|---|---|---|---| +| `dataVolumeSystemTags` | List\ | **广播**到所有数据盘 | 全部数据盘用同一协议时 | +| `dataVolumeSystemTagsOnIndex` | Map\\> | **按 `dataDiskOfferingUuids` 下标精确指定** | 不同数据盘用不同协议时,key = `"0"`/`"1"`/...(与 `dataDiskOfferingUuids` 同序) | + +tag 格式(**注意 `ephemeral::` 前缀必填**):`ephemeral::volumeProtocol::{protocol}`,protocol ∈ §2 枚举。前缀是后端框架的「临时 tag」约定——带前缀的 tag 在建卷时被消费进 `VolumeVO.protocol` 后由框架自动丢弃、绝不落库;漏掉前缀则不被识别(既不消费也不报错,等于没传)。 + +**例:两块数据盘,盘 0 用 Vhost、盘 1 用 CBD** +``` +POST /zstack/v1/vm-instances +{ + "params": { + "name": "vm-1", + "instanceOfferingUuid": "...", + "imageUuid": "...", + "l3NetworkUuids": ["..."], + "primaryStorageUuidForRootVolume": "fbfc8987...", + "dataDiskOfferingUuids": ["offering-A", "offering-B"], + "dataVolumeSystemTagsOnIndex": { + "0": ["ephemeral::volumeProtocol::Vhost"], + "1": ["ephemeral::volumeProtocol::CBD"] + } + } +} +``` +全盘统一协议则改用 `"dataVolumeSystemTags": ["ephemeral::volumeProtocol::Vhost"]`。 + +**语义**: +- tag 在建卷时被**消费**——协议写进 `VolumeVO.protocol`;因是 ephemeral tag,框架**自动不落库**(`createNonInherentSystemTags` 跳过 ephemeral),卷上查不到该 tag,只查 `protocol` 字段。 +- **enum 校验在 API 收到即做**(`APICreateVmInstanceMsg` 拦截器),非法协议(如 `ephemeral::volumeProtocol::BOGUS`)**当场拒绝**,报错见 §7;不会建出一个协议非法的卷。 +- 同一盘若 API 协议参数与 tag 同时存在,API 参数优先(§5 优先级 1 > 2);但 VM 创建链路本就没有 per-盘 API 协议参数,所以 VM 场景 tag 即唯一来源。 +- **注意**:本通道只在 VM 创建链路生效。独立 `APICreateDataVolumeMsg`(§5)请直接用 `protocol` 参数;给它传 `ephemeral::volumeProtocol::` tag 不会被消费成协议(且因 ephemeral 框架不落库,等于丢弃)。 + +## 6. API 三:更改盘的协议 + +**`APIChangeVolumeProtocolMsg`**(action 式) + +``` +PUT /zstack/v1/volumes/{volumeUuid}/actions +{ + "changeVolumeProtocol": { + "protocol": "Vhost" + } +} +``` + +| 参数 | 必填 | 备注 | +|---|---|---| +| volumeUuid | 是 | path 参数 | +| protocol | 是 | 目标协议,§2 枚举 | + +- 返回:`APIChangeVolumeProtocolEvent`,inventory 里带更新后的卷(`protocol` 已翻转)。 +- SDK:`ChangeVolumeProtocolAction`。 +- **离线变更语义**:仅更新元数据;**挂载在 VM 上的卷要求 VM 处于 Stopped**,下次启动 VM 时用新协议挂载。未挂载(Ready 态)的卷可直接切。 +- **UI 门控**:卷挂在非 Stopped 的 VM 上时,"更改协议"入口置灰,tooltip 提示"需要先停止云主机"。 + +## 7. 校验规则与错误文案(真机实测) + +| 场景 | 后端报错(details 关键句) | +|---|---| +| protocol 不在 PS 暴露集合 | `primary storage[uuid:...] does not expose output protocol[NBD]` | +| 创建时传 protocol 但没传 primaryStorageUuid | `primaryStorageUuid is required when protocol is specified` | +| protocol 不是合法枚举 | invalid protocol 类校验错误 | +| 建 VM 时 systemTag 协议非法(`ephemeral::volumeProtocol::BOGUS`) | `unsupported volume protocol[BOGUS]`(API 收到即拒,不建卷) | +| 改协议但 VM 没停机 | VM state 校验错误(要求 Stopped) | +| 卷不在外接(Addon)PS 上 | 仅 external PS 支持协议变更 | + +## 8. UI 建议 + +1. **主存储详情页**:协议 tab / 区块——展示 `outputProtocols` 标签 + `defaultProtocol` 标记 + 「添加协议」按钮(下拉 = 枚举全集 − 已添加集合)。 +2. **创建数据卷表单**:选了 Addon 类型 PS 后出现「协议」下拉(选项 = 该 PS `outputProtocols`,默认选中 `defaultProtocol`);未选 PS 时隐藏该下拉。 +3. **卷列表/详情**:加「协议」列;详情页操作菜单加「更改协议」(门控见 §6)。 +4. **变更协议二次确认**:文案需包含"停机变更、下次启动生效"。 + +## 9. 实现状态总表 + +| 能力 | API 契约 | 后端实现 | 备注 | +|---|---|---|---| +| AddStorageProtocol | ✅ 冻结 | ✅ 完整(注册协议+触发按协议主机准备+写连通性行) | | +| CreateDataVolume.protocol | ✅ 冻结 | ✅ 已实现+真机验证 | | +| ChangeVolumeProtocol | ✅ 冻结 | ✅ 已实现+真机验证(CBD↔Vhost 双向) | | +| per-protocol 主机连通性 + 查询 API | ✅ 冻结(§3.3) | ✅ 新表 + QueryExternalPrimaryStorageHostProtocolRef | 整机状态保持折叠语义;按协议明细独立可查 | +| systemTag 指定协议(建 VM 路径) | ✅ 冻结(§5.1) | ✅ 已实现+IT 验证 | `ephemeral::volumeProtocol::{protocol}`(EphemeralPatternSystemTag)经 `dataVolumeSystemTags` / `dataVolumeSystemTagsOnIndex` 传入;建卷时读进 `VolumeVO.protocol`,框架自动不落库。**仅建 VM 路径**;独立建数据卷 API 用 §5 的 `protocol` 参数 | +| DiskOfferingVO.protocol | 已取消 | — | 产品决定不做 | + +> 联调环境:feature-zbs-vhost 分支构建的 MN(参考 env1 172.24.191.9 / env2 172.24.248.246 部署),三个 API 均可真实调通。 diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsg.java b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsg.java new file mode 100644 index 00000000000..11fb9f9c7ff --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsg.java @@ -0,0 +1,24 @@ +package org.zstack.header.storage.addon.primary; + +import org.springframework.http.HttpMethod; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; + +import java.util.List; + +import static java.util.Arrays.asList; + +@RestRequest( + path = "/external-primary-storage/host-protocol-refs", + optionalPaths = {"/external-primary-storage/{primaryStorageUuid}/host-protocol-refs"}, + method = HttpMethod.GET, + responseClass = APIQueryExternalPrimaryStorageHostProtocolRefReply.class +) +@AutoQuery(replyClass = APIQueryExternalPrimaryStorageHostProtocolRefReply.class, inventoryClass = ExternalPrimaryStorageHostProtocolRefInventory.class) +public class APIQueryExternalPrimaryStorageHostProtocolRefMsg extends APIQueryMessage { + + public static List __example__() { + return asList("primaryStorageUuid=xxx"); + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..7d7a29bd3d5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.header.storage.addon.primary + +import org.zstack.header.storage.addon.primary.APIQueryExternalPrimaryStorageHostProtocolRefReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "QueryExternalPrimaryStorageHostProtocolRef" + + category "storage.primary" + + desc """查询外接主存储按协议的主机连通性""" + + rest { + request { + url "GET /v1/external-primary-storage/host-protocol-refs" + url "GET /v1/external-primary-storage/{primaryStorageUuid}/host-protocol-refs" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryExternalPrimaryStorageHostProtocolRefMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryExternalPrimaryStorageHostProtocolRefReply.class + } + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReply.java b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReply.java new file mode 100644 index 00000000000..333d37c8ca6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReply.java @@ -0,0 +1,39 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.message.DocUtils; +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; + +import java.sql.Timestamp; +import java.util.List; + +import static java.util.Arrays.asList; + +@RestResponse(allTo = "inventories") +public class APIQueryExternalPrimaryStorageHostProtocolRefReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } + + public static APIQueryExternalPrimaryStorageHostProtocolRefReply __example__() { + APIQueryExternalPrimaryStorageHostProtocolRefReply reply = new APIQueryExternalPrimaryStorageHostProtocolRefReply(); + + ExternalPrimaryStorageHostProtocolRefInventory inv = new ExternalPrimaryStorageHostProtocolRefInventory(); + inv.setHostUuid(DocUtils.uuidForAPIDoc()); + inv.setPrimaryStorageUuid(DocUtils.uuidForAPIDoc()); + inv.setProtocol("vhost"); + inv.setStatus("Connected"); + inv.setCreateDate(new Timestamp(DocUtils.date)); + inv.setLastOpDate(new Timestamp(DocUtils.date)); + + reply.setInventories(asList(inv)); + reply.setSuccess(true); + return reply; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReplyDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..5c2c3d0a511 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/APIQueryExternalPrimaryStorageHostProtocolRefReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.storage.addon.primary + +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefInventory + +doc { + + title "查询外接主存储按协议的主机连通性结果" + + field { + name "success" + desc "" + type "boolean" + since "0.6" + } + ref { + name "error" + path "org.zstack.header.storage.addon.primary.APIQueryExternalPrimaryStorageHostProtocolRefReply.error" + desc "错误码,若不为null,则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "2.3.2" + clz ErrorCode.class + } + ref { + name "inventories" + path "org.zstack.header.storage.addon.primary.APIQueryExternalPrimaryStorageHostProtocolRefReply.inventories" + desc "null" + type "List" + since "2.3.2" + clz ExternalPrimaryStorageHostProtocolRefInventory.class + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefInventory.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefInventory.java new file mode 100644 index 00000000000..32a695e4e80 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefInventory.java @@ -0,0 +1,107 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.host.HostInventory; +import org.zstack.header.query.ExpandedQueries; +import org.zstack.header.query.ExpandedQuery; +import org.zstack.header.search.Inventory; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@PythonClassInventory +@Inventory(mappingVOClass = ExternalPrimaryStorageHostProtocolRefVO.class, collectionValueOfMethod = "valueOf1") +@ExpandedQueries({ + @ExpandedQuery(expandedField = "host", inventoryClass = HostInventory.class, + foreignKey = "hostUuid", expandedInventoryKey = "uuid"), + @ExpandedQuery(expandedField = "externalPrimaryStorage", inventoryClass = ExternalPrimaryStorageInventory.class, + foreignKey = "primaryStorageUuid", expandedInventoryKey = "uuid") +}) +public class ExternalPrimaryStorageHostProtocolRefInventory implements Serializable { + private String hostUuid; + + private String primaryStorageUuid; + + private String protocol; + + private String status; + + private Timestamp createDate; + + private Timestamp lastOpDate; + + public ExternalPrimaryStorageHostProtocolRefInventory() { + } + + public ExternalPrimaryStorageHostProtocolRefInventory(ExternalPrimaryStorageHostProtocolRefVO vo) { + this.hostUuid = vo.getHostUuid(); + this.primaryStorageUuid = vo.getPrimaryStorageUuid(); + this.protocol = vo.getProtocol(); + this.status = vo.getStatus() == null ? null : vo.getStatus().toString(); + this.createDate = vo.getCreateDate(); + this.lastOpDate = vo.getLastOpDate(); + } + + public static ExternalPrimaryStorageHostProtocolRefInventory valueOf(ExternalPrimaryStorageHostProtocolRefVO vo) { + return new ExternalPrimaryStorageHostProtocolRefInventory(vo); + } + + public static List valueOf1(Collection vos) { + List invs = new ArrayList<>(); + for (ExternalPrimaryStorageHostProtocolRefVO vo : vos) { + invs.add(valueOf(vo)); + } + return invs; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO.java new file mode 100644 index 00000000000..579998efe8a --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO.java @@ -0,0 +1,110 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.host.HostEO; +import org.zstack.header.host.HostVO; +import org.zstack.header.storage.primary.PrimaryStorageEO; +import org.zstack.header.storage.primary.PrimaryStorageHostStatus; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.header.vo.EntityGraph; +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.SoftDeletionCascade; +import org.zstack.header.vo.SoftDeletionCascades; + +import javax.persistence.*; +import java.sql.Timestamp; + +@Entity +@Table +@SoftDeletionCascades({ + @SoftDeletionCascade(parent = PrimaryStorageVO.class, joinColumn = "primaryStorageUuid"), + @SoftDeletionCascade(parent = HostVO.class, joinColumn = "hostUuid") +}) +@EntityGraph( + friends = { + @EntityGraph.Neighbour(type = PrimaryStorageVO.class, myField = "primaryStorageUuid", targetField = "uuid"), + @EntityGraph.Neighbour(type = HostVO.class, myField = "hostUuid", targetField = "uuid"), + } +) +public class ExternalPrimaryStorageHostProtocolRefVO { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column + private long id; + + @Column + @ForeignKey(parentEntityClass = HostEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String hostUuid; + + @Column + @ForeignKey(parentEntityClass = PrimaryStorageEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String primaryStorageUuid; + + @Column + private String protocol; + + @Column + @Enumerated(EnumType.STRING) + private PrimaryStorageHostStatus status; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public PrimaryStorageHostStatus getStatus() { + return status; + } + + public void setStatus(PrimaryStorageHostStatus status) { + this.status = status; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO_.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO_.java new file mode 100644 index 00000000000..199a625ef24 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostProtocolRefVO_.java @@ -0,0 +1,18 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.storage.primary.PrimaryStorageHostStatus; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(ExternalPrimaryStorageHostProtocolRefVO.class) +public class ExternalPrimaryStorageHostProtocolRefVO_ { + public static volatile SingularAttribute id; + public static volatile SingularAttribute hostUuid; + public static volatile SingularAttribute primaryStorageUuid; + public static volatile SingularAttribute protocol; + public static volatile SingularAttribute status; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO.java index 035f89b1ff2..9cc0374b269 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO.java @@ -11,20 +11,9 @@ @Table @PrimaryKeyJoinColumn(name = "id", referencedColumnName = "id") public class ExternalPrimaryStorageHostRefVO extends PrimaryStorageHostRefVO { - @Column - private String protocol; - @Column private int hostId; - public String getProtocol() { - return protocol; - } - - public void setProtocol(String protocol) { - this.protocol = protocol; - } - public int getHostId() { return hostId; } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO_.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO_.java index 0701135df99..2411be7c4e0 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO_.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageHostRefVO_.java @@ -8,5 +8,4 @@ @StaticMetamodel(ExternalPrimaryStorageHostRefVO.class) public class ExternalPrimaryStorageHostRefVO_ extends PrimaryStorageHostRefVO_ { public static volatile SingularAttribute hostId; - public static volatile SingularAttribute protocol; } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index feb4c5c11c7..d1644d8da5a 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -49,6 +49,15 @@ public interface PrimaryStorageControllerSvc { void reportNodeHealthy(HostInventory host, ReturnValueCompletion comp); StorageCapabilities reportCapabilities(); + /** + * called after a new output protocol is added on the storage; implementations + * may prepare the given connected hosts for the protocol, e.g. deploy a + * data-path target on them + */ + default void onProtocolAdded(String protocol, List hosts, Completion completion) { + completion.success(); + } + // TODO: remove this method in future @Deprecated String allocateSpace(AllocateSpaceSpec aspec); diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolEvent.java b/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolEvent.java new file mode 100644 index 00000000000..5bc67c01fa7 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolEvent.java @@ -0,0 +1,30 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.volume.VolumeInventory; + +@RestResponse(allTo = "inventory") +public class APIChangeVolumeProtocolEvent extends APIEvent { + private VolumeInventory inventory; + + public APIChangeVolumeProtocolEvent() { + } + + public APIChangeVolumeProtocolEvent(String apiId) { + super(apiId); + } + + public VolumeInventory getInventory() { + return inventory; + } + + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } + + public static APIChangeVolumeProtocolEvent __example__() { + APIChangeVolumeProtocolEvent event = new APIChangeVolumeProtocolEvent(); + return event; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolMsg.java b/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolMsg.java new file mode 100644 index 00000000000..fb9012e5351 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIChangeVolumeProtocolMsg.java @@ -0,0 +1,47 @@ +package org.zstack.header.storage.primary; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.volume.VolumeMessage; +import org.zstack.header.volume.VolumeProtocol; +import org.zstack.header.volume.VolumeVO; + +@RestRequest( + path = "/volumes/{volumeUuid}/actions", + responseClass = APIChangeVolumeProtocolEvent.class, + method = HttpMethod.PUT, + isAction = true +) +public class APIChangeVolumeProtocolMsg extends APIMessage implements VolumeMessage { + @APIParam(resourceType = VolumeVO.class, operationTarget = true) + private String volumeUuid; + + @APIParam + private String protocol; + + @Override + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public static APIChangeVolumeProtocolMsg __example__() { + APIChangeVolumeProtocolMsg msg = new APIChangeVolumeProtocolMsg(); + msg.setVolumeUuid(uuid()); + msg.setProtocol(VolumeProtocol.Vhost.toString()); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/UpdatePrimaryStorageHostStatusMsg.java b/header/src/main/java/org/zstack/header/storage/primary/UpdatePrimaryStorageHostStatusMsg.java index d7a2e8720d0..bac5eefd129 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/UpdatePrimaryStorageHostStatusMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/UpdatePrimaryStorageHostStatusMsg.java @@ -11,6 +11,9 @@ public class UpdatePrimaryStorageHostStatusMsg extends NeedReplyMessage implemen private String hostUuid; private PrimaryStorageHostStatus status; private ErrorCode reason; + // null means the host-level default data path; a non-null value addresses + // the per-protocol connectivity row of an external primary storage + private String protocol; @Override public String getPrimaryStorageUuid() { @@ -44,4 +47,12 @@ public ErrorCode getReason() { public void setReason(ErrorCode reason) { this.reason = reason; } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } } diff --git a/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java index 9fbb753f19d..cee141c0ca8 100755 --- a/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java @@ -72,6 +72,9 @@ public class APICreateDataVolumeMsg extends APICreateMessage implements APIAudit @APIParam(required = false, resourceType = PrimaryStorageVO.class) private String primaryStorageUuid; + @APIParam(required = false) + private String protocol; + public String getPrimaryStorageUuid() { return primaryStorageUuid; } @@ -80,6 +83,14 @@ public void setPrimaryStorageUuid(String primaryStorageUuid) { this.primaryStorageUuid = primaryStorageUuid; } + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + public String getName() { return name; } diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeMsg.java index 833df491428..54b9196ccec 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeMsg.java @@ -10,6 +10,7 @@ public class CreateDataVolumeMsg extends NeedReplyMessage implements VolumeCreat private String primaryStorageUuid; private String accountUuid; private String resourceUuid; + private String protocol; private APICreateDataVolumeMsg apiMsg; public String getPrimaryStorageUuid() { @@ -20,6 +21,14 @@ public void setPrimaryStorageUuid(String primaryStorageUuid) { this.primaryStorageUuid = primaryStorageUuid; } + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + public String getName() { return name; } diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index aa25ed03592..814c01c8345 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -234,6 +234,9 @@ private void checkHostStatus(KVMHostInventory host, List(compl) { @Override public void success(NodeHealthy returnValue) { + returnValue.getHealthy().forEach((p, h) -> updateHostProtocolRef(host.getUuid(), extPs.getUuid(), + p.toString(), h == StorageHealthy.Ok ? PrimaryStorageHostStatus.Connected : PrimaryStorageHostStatus.Disconnected)); + ErrorCode err = null; PrimaryStorageHostStatus status; // TODO add multi protocol support @@ -277,6 +280,16 @@ public void run(MessageReply reply) { }).run(completion); } + private void updateHostProtocolRef(String hostUuid, String psUuid, String protocol, PrimaryStorageHostStatus status) { + UpdatePrimaryStorageHostStatusMsg msg = new UpdatePrimaryStorageHostStatusMsg(); + msg.setPrimaryStorageUuid(psUuid); + msg.setHostUuid(hostUuid); + msg.setProtocol(protocol); + msg.setStatus(status); + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, psUuid); + bus.send(msg); + } + @Override public List getStoragePathsForVolumeSync(HostInventory host, PrimaryStorageInventory attachedPs) { if (!PrimaryStorageConstant.EXTERNAL_PRIMARY_STORAGE_TYPE.equals(attachedPs.getType())) { diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalProperty.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalProperty.java index 8e71a24def1..6040ec067e0 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalProperty.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalProperty.java @@ -25,8 +25,16 @@ public class ZbsGlobalProperty { public static List MN_NETWORKS; @GlobalProperty(name="Zbs.vhost.targetImage", defaultValue = "zbs-vhost:latest") public static String VHOST_TARGET_IMAGE; - @GlobalProperty(name="Zbs.vhost.targetCores", defaultValue = "[0,1]") + // empty -> agent computes cores from host cpu count (VHOST_TARGET_CORE_COUNT); set to override e.g. "[0,1]" + @GlobalProperty(name="Zbs.vhost.targetCores", defaultValue = "") public static String VHOST_TARGET_CORES; + @GlobalProperty(name="Zbs.vhost.targetCoreCount", defaultValue = "2") + public static int VHOST_TARGET_CORE_COUNT; @GlobalProperty(name="Zbs.vhost.hugepageNr", defaultValue = "256") public static int VHOST_HUGEPAGE_NR; + // image delivery: local tar shipped onto the host wins; else agent downloads from this url. either may be empty. + @GlobalProperty(name="Zbs.vhost.targetImageTar", defaultValue = "") + public static String VHOST_TARGET_IMAGE_TAR; + @GlobalProperty(name="Zbs.vhost.targetImageUrl", defaultValue = "") + public static String VHOST_TARGET_IMAGE_URL; } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index c89deb0affa..6ec44102ed9 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -15,6 +15,8 @@ import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO; +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO_; import org.zstack.core.db.SQL; import org.zstack.core.workflow.FlowChainBuilder; import org.zstack.core.workflow.ShareFlow; @@ -112,6 +114,7 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar public static final String GET_VOLUME_CLIENTS_PATH = "/zbs/primarystorage/volume/clients"; public static final String UPDATE_HOST_DEPENDENCY_PATH = "/zbs/primarystorage/host/updatedependency"; public static final String VHOST_TARGET_ENSURE_PATH = "/zbs/primarystorage/vhost/target/ensure"; + public static final String VHOST_TARGET_HEALTH_PATH = "/zbs/primarystorage/vhost/target/health"; public static final String VHOST_ACTIVATE_PATH = "/zbs/primarystorage/vhost/activate"; public static final String VHOST_DEACTIVATE_PATH = "/zbs/primarystorage/vhost/deactivate"; public static final String VHOST_RESIZE_PATH = "/zbs/primarystorage/vhost/resize"; @@ -191,8 +194,12 @@ public void run(MessageReply reply) { private void fillVhostTargetParams(VhostActivateCmd cmd) { cmd.image = ZbsGlobalProperty.VHOST_TARGET_IMAGE; - cmd.cores = ZbsGlobalProperty.VHOST_TARGET_CORES; + // empty cores lets the agent compute them from the host cpu count + cmd.cores = StringUtils.isEmpty(ZbsGlobalProperty.VHOST_TARGET_CORES) ? null : ZbsGlobalProperty.VHOST_TARGET_CORES; + cmd.coreCount = ZbsGlobalProperty.VHOST_TARGET_CORE_COUNT; cmd.hugepageNr = ZbsGlobalProperty.VHOST_HUGEPAGE_NR; + cmd.imageTar = StringUtils.isEmpty(ZbsGlobalProperty.VHOST_TARGET_IMAGE_TAR) ? null : ZbsGlobalProperty.VHOST_TARGET_IMAGE_TAR; + cmd.imageUrl = StringUtils.isEmpty(ZbsGlobalProperty.VHOST_TARGET_IMAGE_URL) ? null : ZbsGlobalProperty.VHOST_TARGET_IMAGE_URL; cmd.socketDir = ZbsConstants.VHOST_SOCKET_DIR; } @@ -919,11 +926,112 @@ public void run(MessageReply reply) { CheckHostStorageConnectionRsp rsp = hreply.toResponse(CheckHostStorageConnectionRsp.class); NodeHealthy healthy = new NodeHealthy(); healthy.setHealthy(VolumeProtocol.CBD, rsp.isSuccess() ? StorageHealthy.Ok : StorageHealthy.Failed); + + // vhost volumes don't ride the cbd heartbeat; their data path is the + // host's SPDK target container, so it needs its own health probe. + if (!supportsVhost()) { + comp.success(healthy); + return; + } + checkVhostTargetHealthy(host, healthy, comp); + } + }); + } + + private boolean supportsVhost() { + return Q.New(PrimaryStorageOutputProtocolRefVO.class) + .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, self.getUuid()) + .eq(PrimaryStorageOutputProtocolRefVO_.outputProtocol, VolumeProtocol.Vhost.toString()) + .isExists(); + } + + private void checkVhostTargetHealthy(HostInventory host, NodeHealthy healthy, ReturnValueCompletion comp) { + VhostTargetHealthCmd cmd = new VhostTargetHealthCmd(); + cmd.socketDir = ZbsConstants.VHOST_SOCKET_DIR; + + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setCommand(cmd); + msg.setHostUuid(host.getUuid()); + msg.setPath(VHOST_TARGET_HEALTH_PATH); + msg.setNoStatusCheck(true); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(msg, new CloudBusCallBack(comp) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + healthy.setHealthy(VolumeProtocol.Vhost, StorageHealthy.Failed); + comp.success(healthy); + return; + } + + KVMHostAsyncHttpCallReply hreply = reply.castReply(); + VhostTargetHealthRsp rsp = hreply.toResponse(VhostTargetHealthRsp.class); + healthy.setHealthy(VolumeProtocol.Vhost, + rsp.isSuccess() && rsp.healthy ? StorageHealthy.Ok : StorageHealthy.Failed); comp.success(healthy); } }); } + @Override + public void onProtocolAdded(String protocol, List hosts, Completion completion) { + if (!VolumeProtocol.Vhost.toString().equals(protocol)) { + completion.success(); + return; + } + + new While<>(hosts).each(this::ensureVhostTargetOnHost).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().isEmpty()) { + completion.success(); + } else { + completion.fail(errorCodeList.getCauses().get(0)); + } + } + }); + } + + private void ensureVhostTargetOnHost(HostInventory host, WhileCompletion completion) { + VhostActivateCmd cmd = new VhostActivateCmd(); + fillVhostTargetParams(cmd); + + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setCommand(cmd); + msg.setHostUuid(host.getUuid()); + msg.setPath(VHOST_TARGET_ENSURE_PATH); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + PrimaryStorageHostStatus status = PrimaryStorageHostStatus.Connected; + if (!reply.isSuccess()) { + status = PrimaryStorageHostStatus.Disconnected; + completion.addError(reply.getError()); + } else { + KVMHostAsyncHttpCallReply hreply = reply.castReply(); + CheckHostStorageConnectionRsp rsp = hreply.toResponse(CheckHostStorageConnectionRsp.class); + if (!rsp.isSuccess()) { + status = PrimaryStorageHostStatus.Disconnected; + completion.addError(operr(ORG_ZSTACK_STORAGE_ZBS_10006, "failed to ensure vhost target on host[%s]: %s", + host.getUuid(), rsp.getError())); + } + } + + // fire-and-forget: the handler runs after the in-flight protocol + // operation releases the primary storage queue + UpdatePrimaryStorageHostStatusMsg umsg = new UpdatePrimaryStorageHostStatusMsg(); + umsg.setPrimaryStorageUuid(self.getUuid()); + umsg.setHostUuid(host.getUuid()); + umsg.setProtocol(VolumeProtocol.Vhost.toString()); + umsg.setStatus(status); + bus.makeTargetServiceIdByResourceUuid(umsg, PrimaryStorageConstant.SERVICE_ID, self.getUuid()); + bus.send(umsg); + completion.done(); + } + }); + } + @Override public StorageCapabilities reportCapabilities() { return capabilities; @@ -2183,16 +2291,27 @@ public static class UpdateHostDependencyRsp extends AgentResponse { public static class VhostActivateCmd extends AgentCommand { public String image; public String cores; + public Integer coreCount; public Integer hugepageNr; public String socketDir; public String controlSock; public String clientConf; public String imageTar; + public String imageUrl; public String installPath; public String controllerName; public String bdevName; } + public static class VhostTargetHealthCmd extends AgentCommand { + public String socketDir; + public String controlSock; + } + + public static class VhostTargetHealthRsp extends AgentResponse { + public boolean healthy; + } + public static class VhostActivateRsp extends AgentResponse { private String socketPath; diff --git a/sdk/src/main/java/SourceClassMap.java b/sdk/src/main/java/SourceClassMap.java index 63756cc5fa5..5ba5caf4248 100644 --- a/sdk/src/main/java/SourceClassMap.java +++ b/sdk/src/main/java/SourceClassMap.java @@ -371,6 +371,7 @@ public class SourceClassMap { put("org.zstack.header.sshkeypair.SshKeyPairInventory", "org.zstack.sdk.SshKeyPairInventory"); put("org.zstack.header.sshkeypair.SshPrivateKeyPairInventory", "org.zstack.sdk.SshPrivateKeyPairInventory"); put("org.zstack.header.storage.addon.backup.ExternalBackupStorageInventory", "org.zstack.sdk.ExternalBackupStorageInventory"); + put("org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefInventory", "org.zstack.sdk.ExternalPrimaryStorageHostProtocolRefInventory"); put("org.zstack.header.storage.addon.primary.ExternalPrimaryStorageInventory", "org.zstack.sdk.ExternalPrimaryStorageInventory"); put("org.zstack.header.storage.backup.BackupMode", "org.zstack.sdk.BackupMode"); put("org.zstack.header.storage.backup.BackupStorageInventory", "org.zstack.sdk.BackupStorageInventory"); @@ -1103,6 +1104,7 @@ public class SourceClassMap { put("org.zstack.sdk.ExternalBackupInventory", "org.zstack.externalbackup.ExternalBackupInventory"); put("org.zstack.sdk.ExternalBackupState", "org.zstack.externalbackup.ExternalBackupState"); put("org.zstack.sdk.ExternalBackupStorageInventory", "org.zstack.header.storage.addon.backup.ExternalBackupStorageInventory"); + put("org.zstack.sdk.ExternalPrimaryStorageHostProtocolRefInventory", "org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefInventory"); put("org.zstack.sdk.ExternalPrimaryStorageInventory", "org.zstack.header.storage.addon.primary.ExternalPrimaryStorageInventory"); put("org.zstack.sdk.ExternalServiceCapabilities", "org.zstack.header.core.external.service.ExternalServiceCapabilities"); put("org.zstack.sdk.ExternalServiceCapabilitiesBuilder", "org.zstack.core.externalservice.ExternalServiceCapabilitiesBuilder"); diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolAction.java b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolAction.java new file mode 100644 index 00000000000..9c6de102065 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class ChangeVolumeProtocolAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.ChangeVolumeProtocolResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String volumeUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String protocol; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.ChangeVolumeProtocolResult value = res.getResult(org.zstack.sdk.ChangeVolumeProtocolResult.class); + ret.value = value == null ? new org.zstack.sdk.ChangeVolumeProtocolResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/volumes/{volumeUuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "changeVolumeProtocol"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolResult.java b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolResult.java new file mode 100644 index 00000000000..e1bcf6a2185 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeProtocolResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + +import org.zstack.sdk.VolumeInventory; + +public class ChangeVolumeProtocolResult { + public VolumeInventory inventory; + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } + public VolumeInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/CreateDataVolumeAction.java b/sdk/src/main/java/org/zstack/sdk/CreateDataVolumeAction.java index 190079d6da1..23b06c308b5 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateDataVolumeAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateDataVolumeAction.java @@ -40,6 +40,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String primaryStorageUuid; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String protocol; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/ExternalPrimaryStorageHostProtocolRefInventory.java b/sdk/src/main/java/org/zstack/sdk/ExternalPrimaryStorageHostProtocolRefInventory.java new file mode 100644 index 00000000000..cb89ce7dca3 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/ExternalPrimaryStorageHostProtocolRefInventory.java @@ -0,0 +1,55 @@ +package org.zstack.sdk; + + + +public class ExternalPrimaryStorageHostProtocolRefInventory { + + public java.lang.String hostUuid; + public void setHostUuid(java.lang.String hostUuid) { + this.hostUuid = hostUuid; + } + public java.lang.String getHostUuid() { + return this.hostUuid; + } + + public java.lang.String primaryStorageUuid; + public void setPrimaryStorageUuid(java.lang.String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + public java.lang.String getPrimaryStorageUuid() { + return this.primaryStorageUuid; + } + + public java.lang.String protocol; + public void setProtocol(java.lang.String protocol) { + this.protocol = protocol; + } + public java.lang.String getProtocol() { + return this.protocol; + } + + public java.lang.String status; + public void setStatus(java.lang.String status) { + this.status = status; + } + public java.lang.String getStatus() { + return this.status; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefAction.java b/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefAction.java new file mode 100644 index 00000000000..a6dd708f07e --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryExternalPrimaryStorageHostProtocolRefAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefResult value = res.getResult(org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefResult.class); + ret.value = value == null ? new org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/external-primary-storage/host-protocol-refs"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefResult.java b/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefResult.java new file mode 100644 index 00000000000..f7d9908a7c0 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryExternalPrimaryStorageHostProtocolRefResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk; + + + +public class QueryExternalPrimaryStorageHostProtocolRefResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 921c8431326..3d04cb00c18 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -316,6 +316,12 @@ protected void handle(APICleanUpImageCacheOnPrimaryStorageMsg msg) { @Override protected void handle(UpdatePrimaryStorageHostStatusMsg msg) { + if (msg.getProtocol() != null) { + updateHostProtocolStatus(msg.getPrimaryStorageUuid(), msg.getHostUuid(), msg.getProtocol(), msg.getStatus()); + bus.reply(msg, new UpdatePrimaryStorageHostStatusReply()); + return; + } + ExternalPrimaryStorageHostRefVO ref = Q.New(ExternalPrimaryStorageHostRefVO.class) .eq(ExternalPrimaryStorageHostRefVO_.hostUuid, msg.getHostUuid()) .eq(ExternalPrimaryStorageHostRefVO_.primaryStorageUuid, msg.getPrimaryStorageUuid()) @@ -331,6 +337,29 @@ protected void handle(UpdatePrimaryStorageHostStatusMsg msg) { bus.reply(msg, reply); } + private void updateHostProtocolStatus(String psUuid, String hostUuid, String protocol, PrimaryStorageHostStatus newStatus) { + ExternalPrimaryStorageHostProtocolRefVO ref = Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, psUuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, hostUuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, protocol) + .find(); + if (ref == null) { + ref = new ExternalPrimaryStorageHostProtocolRefVO(); + ref.setPrimaryStorageUuid(psUuid); + ref.setHostUuid(hostUuid); + ref.setProtocol(protocol); + ref.setStatus(newStatus); + dbf.persist(ref); + logger.debug(String.format("created protocol[%s] connectivity row between primary storage[uuid:%s]" + + " and host[uuid:%s] with status %s", protocol, psUuid, hostUuid, newStatus)); + } else if (ref.getStatus() != newStatus) { + ref.setStatus(newStatus); + dbf.update(ref); + logger.debug(String.format("change protocol[%s] connectivity between primary storage[uuid:%s]" + + " and host[uuid:%s] to %s", protocol, psUuid, hostUuid, newStatus)); + } + } + @Override protected void handle(InstantiateVolumeOnPrimaryStorageMsg msg) { VolumeInventory volume = msg.getVolume(); @@ -2381,6 +2410,32 @@ protected void doAddProtocol(APIAddStorageProtocolMsg msg, Completion completion storageVO.getOutputProtocols().add(ref); dbf.updateAndRefresh(storageVO); } - super.doAddProtocol(msg, completion); + + List hostVOs = SQL.New("select h from HostVO h, PrimaryStorageClusterRefVO ref" + + " where h.clusterUuid = ref.clusterUuid" + + " and ref.primaryStorageUuid = :psUuid" + + " and h.status = :hostStatus", HostVO.class) + .param("psUuid", msg.getUuid()) + .param("hostStatus", HostStatus.Connected) + .list(); + if (hostVOs.isEmpty()) { + super.doAddProtocol(msg, completion); + return; + } + + controller.onProtocolAdded(msg.getOutputProtocol(), HostInventory.valueOf(hostVOs), new Completion(completion) { + @Override + public void success() { + ExternalPrimaryStorage.super.doAddProtocol(msg, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + // the protocol is already registered; host preparation self-heals on the next ping + logger.warn(String.format("failed to prepare hosts for protocol[%s] on primary storage[uuid:%s]: %s", + msg.getOutputProtocol(), msg.getUuid(), errorCode.getDetails())); + ExternalPrimaryStorage.super.doAddProtocol(msg, completion); + } + }); } } diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java index 11c09d144cb..83b737489c2 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java @@ -25,6 +25,11 @@ import org.zstack.header.storage.snapshot.group.APIRevertVmFromSnapshotGroupMsg; import org.zstack.header.volume.APICreateVolumeSnapshotGroupMsg; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.VmInstanceState; import org.zstack.header.zone.ZoneVO; import org.zstack.header.zone.ZoneVO_; import org.zstack.utils.network.NetworkUtils; diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java b/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java index f9e18f0a1e5..d06244e9204 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java @@ -42,6 +42,12 @@ import org.zstack.header.storage.primary.PrimaryStorageHostStatus; import org.zstack.header.storage.primary.PrimaryStorageVO; import org.zstack.header.storage.primary.PrimaryStorageVO_; +import org.zstack.header.storage.primary.APIChangeVolumeProtocolMsg; +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO; +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO_; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO_; +import org.zstack.header.volume.VolumeProtocol; import org.zstack.header.storage.snapshot.ConsistentType; import org.zstack.header.storage.snapshot.VolumeSnapshotTreeVO; import org.zstack.header.storage.snapshot.VolumeSnapshotTreeVO_; @@ -131,6 +137,8 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIDeleteDataVolumeMsg) msg); } else if (msg instanceof APICreateDataVolumeMsg) { validate((APICreateDataVolumeMsg) msg); + } else if (msg instanceof APIChangeVolumeProtocolMsg) { + validate((APIChangeVolumeProtocolMsg) msg); } else if (msg instanceof APIBackupDataVolumeMsg) { validate((APIBackupDataVolumeMsg) msg); } else if (msg instanceof APIAttachDataVolumeToVmMsg) { @@ -448,6 +456,20 @@ private ErrorCode checkHostAccessible(VolumeVO volumeVO, String hostUuid) { return null; } + if (volumeVO.getProtocol() != null) { + PrimaryStorageHostStatus protocolStatus = Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, hostUuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, volumeVO.getPrimaryStorageUuid()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, volumeVO.getProtocol()) + .select(ExternalPrimaryStorageHostProtocolRefVO_.status) + .findValue(); + if (protocolStatus == PrimaryStorageHostStatus.Disconnected) { + return operr(ORG_ZSTACK_STORAGE_VOLUME_10066, "Can not attach volume to vm runs on host[uuid: %s] whose protocol[%s] " + + "is disconnected with volume's storage[uuid: %s]", hostUuid, volumeVO.getProtocol(), volumeVO.getPrimaryStorageUuid()); + } + return null; + } + PrimaryStorageHostStatus primaryStorageHostStatus = Q.New(PrimaryStorageHostRefVO.class) .eq(PrimaryStorageHostRefVO_.hostUuid, hostUuid) .eq(PrimaryStorageHostRefVO_.primaryStorageUuid, volumeVO.getPrimaryStorageUuid()) @@ -478,6 +500,52 @@ private void validate(APICreateDataVolumeMsg msg) { Long diskSize = Q.New(DiskOfferingVO.class).eq(DiskOfferingVO_.uuid, msg.getDiskOfferingUuid()).select(DiskOfferingVO_.diskSize).findValue(); msg.setDiskSize(diskSize); } + + validateVolumeProtocol(msg.getProtocol(), msg.getPrimaryStorageUuid()); + } + + // a chosen protocol is only meaningful against a specific primary storage that + // exposes it; mirror APIChangeVolumeProtocolMsg so create and change agree. + private void validateVolumeProtocol(String protocol, String primaryStorageUuid) { + if (protocol == null) { + return; + } + boolean known = Arrays.stream(VolumeProtocol.values()).anyMatch(p -> p.name().equals(protocol)); + if (!known) { + throw new ApiMessageInterceptionException(argerr("unsupported volume protocol[%s]", protocol)); + } + if (primaryStorageUuid == null) { + throw new ApiMessageInterceptionException(argerr("primaryStorageUuid is required when protocol[%s] is specified", protocol)); + } + if (!Q.New(PrimaryStorageOutputProtocolRefVO.class) + .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, primaryStorageUuid) + .eq(PrimaryStorageOutputProtocolRefVO_.outputProtocol, protocol) + .isExists()) { + throw new ApiMessageInterceptionException(argerr("primary storage[uuid:%s] does not expose output protocol[%s]", primaryStorageUuid, protocol)); + } + } + + private void validate(APIChangeVolumeProtocolMsg msg) { + VolumeVO vol = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (vol.getPrimaryStorageUuid() == null) { + throw new ApiMessageInterceptionException(argerr("volume[uuid:%s] is not on any primary storage, cannot change its protocol", msg.getVolumeUuid())); + } + if (msg.getProtocol().equals(vol.getProtocol())) { + throw new ApiMessageInterceptionException(argerr("volume[uuid:%s] already uses output protocol[%s]", msg.getVolumeUuid(), msg.getProtocol())); + } + validateVolumeProtocol(msg.getProtocol(), vol.getPrimaryStorageUuid()); + + // offline switch only: a volume attached to a running vm would have its live + // qemu disk diverge from the persisted protocol until the next start. + if (vol.getVmInstanceUuid() != null) { + VmInstanceState vmState = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vol.getVmInstanceUuid()) + .select(VmInstanceVO_.state).findValue(); + if (vmState != null && vmState != VmInstanceState.Stopped) { + throw new ApiMessageInterceptionException(argerr("volume[uuid:%s] is attached to vm[uuid:%s] in state[%s]; stop the vm before changing protocol", + msg.getVolumeUuid(), vol.getVmInstanceUuid(), vmState)); + } + } } private void validate(APIDeleteDataVolumeMsg msg) { @@ -662,8 +730,31 @@ private void validate(APIUndoSnapshotCreationMsg msg) { } } + private void validateDataVolumeProtocolSystemTags(APICreateVmInstanceMsg msg) { + List tags = new ArrayList<>(); + if (msg.getDataVolumeSystemTags() != null) { + tags.addAll(msg.getDataVolumeSystemTags()); + } + if (msg.getDataVolumeSystemTagsOnIndex() != null) { + msg.getDataVolumeSystemTagsOnIndex().values().forEach(tags::addAll); + } + + for (String tag : tags) { + if (!VolumeSystemTags.VOLUME_PROTOCOL.isMatch(tag)) { + continue; + } + String protocol = VolumeSystemTags.VOLUME_PROTOCOL.getTokenByTag(tag, VolumeSystemTags.VOLUME_PROTOCOL_TOKEN); + boolean known = Arrays.stream(VolumeProtocol.values()).anyMatch(p -> p.name().equals(protocol)); + if (!known) { + throw new ApiMessageInterceptionException(argerr("unsupported volume protocol[%s]", protocol)); + } + } + } + @Transactional protected void validate(APICreateVmInstanceMsg msg) { + validateDataVolumeProtocolSystemTags(msg); + if (CollectionUtils.isEmpty(msg.getDiskAOs())) { return; } diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index 85efec20c2d..55e5664aab1 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -2396,6 +2396,8 @@ public String getName() { private void handleApiMessage(APIMessage msg) { if (msg instanceof APIChangeVolumeStateMsg) { handle((APIChangeVolumeStateMsg) msg); + } else if (msg instanceof APIChangeVolumeProtocolMsg) { + handle((APIChangeVolumeProtocolMsg) msg); } else if (msg instanceof APICreateVolumeSnapshotMsg) { handle((APICreateVolumeSnapshotMsg) msg); } else if (msg instanceof APICreateVolumeSnapshotGroupMsg) { @@ -3629,6 +3631,25 @@ private void handle(APIChangeVolumeStateMsg msg) { bus.publish(evt); } + private void handle(APIChangeVolumeProtocolMsg msg) { + APIChangeVolumeProtocolEvent evt = new APIChangeVolumeProtocolEvent(msg.getId()); + // offline switch: persist the new protocol; the next activate (on vm start) + // builds the active path for it. a 0-row update means the volume vanished + // after interception, so fail instead of reporting a no-op success. + int updated = SQL.New("update VolumeVO vol set vol.protocol = :protocol where vol.uuid = :uuid") + .param("protocol", msg.getProtocol()) + .param("uuid", msg.getVolumeUuid()) + .execute(); + if (updated == 0) { + evt.setError(operr("volume[uuid:%s] no longer exists, cannot change its protocol", msg.getVolumeUuid())); + bus.publish(evt); + return; + } + self = dbf.reload(self); + evt.setInventory(VolumeInventory.valueOf(self)); + bus.publish(evt); + } + class VolumeSize { long size; long actualSize; diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java index 248ff279765..eadfc20e126 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java @@ -616,6 +616,13 @@ private VolumeInventory createVolume(CreateVolumeMsg msg) { } } + if (vo.getProtocol() == null) { + String protocolTag = msg.getSystemTag(VolumeSystemTags.VOLUME_PROTOCOL::isMatch); + if (protocolTag != null) { + vo.setProtocol(VolumeSystemTags.VOLUME_PROTOCOL.getTokenByTag(protocolTag, VolumeSystemTags.VOLUME_PROTOCOL_TOKEN)); + } + } + List exts = pluginRgty.getExtensionList(CreateDataVolumeExtensionPoint.class); for (CreateDataVolumeExtensionPoint ext : exts) { ext.beforeCreateVolume(VolumeInventory.valueOf(vo)); @@ -1030,6 +1037,7 @@ private void handle(CreateDataVolumeMsg msg) { vo.setType(VolumeType.Data); vo.setStatus(VolumeStatus.NotInstantiated); vo.setAccountUuid(msg.getAccountUuid()); + vo.setProtocol(msg.getProtocol()); if (msg.getSystemTags() != null) { Iterator iterators = msg.getSystemTags().iterator(); @@ -1123,6 +1131,7 @@ private void handle(APICreateDataVolumeMsg msg) { cmsg.setDiskOfferingUuid(msg.getDiskOfferingUuid()); cmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); cmsg.setDescription(msg.getDescription()); + cmsg.setProtocol(msg.getProtocol()); cmsg.setApiMsg(msg); bus.makeLocalServiceId(cmsg, VolumeConstant.SERVICE_ID); bus.send(cmsg, new CloudBusCallBack(msg) { diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java index d20cebd9592..1b03979e15b 100644 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java @@ -5,6 +5,7 @@ import org.zstack.header.tag.TagDefinition; import org.zstack.header.volume.VolumeVO; import org.zstack.tag.EphemeralSystemTag; +import org.zstack.tag.EphemeralPatternSystemTag; import org.zstack.tag.PatternedSystemTag; import org.zstack.tag.SystemTag; @@ -51,5 +52,11 @@ public class VolumeSystemTags { public static String VOLUME_QOS_TOKEN = "qos"; public static PatternedSystemTag VOLUME_QOS = new PatternedSystemTag(String.format("%s::{%s}", VOLUME_QOS_TOKEN, VOLUME_QOS_TOKEN), VolumeVO.class); + // consumed at volume creation: read into VolumeVO.protocol. ephemeral so the + // framework never persists it as a resident tag (createNonInherentSystemTags + // skips ephemeral tags), matching the consume-once intent. + public static String VOLUME_PROTOCOL_TOKEN = "protocol"; + public static EphemeralPatternSystemTag VOLUME_PROTOCOL = new EphemeralPatternSystemTag(String.format("volumeProtocol::{%s}", VOLUME_PROTOCOL_TOKEN), VolumeVO.class); + public static SystemTag FAST_REVERT = new SystemTag("fast::revert", VolumeVO.class); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy index 84401fc4d0c..e5556518c71 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy @@ -2,9 +2,18 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.cloudbus.CloudBusCallBack import org.zstack.core.db.Q +import org.zstack.core.db.SQL +import org.zstack.header.identity.AccountConstant +import org.zstack.header.message.MessageReply +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO_ import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO_ +import org.zstack.header.storage.primary.PrimaryStorageHostRefVO +import org.zstack.header.storage.primary.PrimaryStorageHostRefVO_ +import org.zstack.header.storage.primary.PrimaryStorageHostStatus import org.zstack.header.storage.backup.UploadImageToRemoteTargetMsg import org.zstack.header.storage.backup.UploadImageToRemoteTargetReply import org.zstack.header.vm.VmInstanceState @@ -12,6 +21,11 @@ import org.zstack.header.volume.VolumeAO_ import org.zstack.header.volume.VolumeProtocol import org.zstack.header.volume.VolumeVO import org.zstack.header.volume.VolumeVO_ +import org.zstack.header.volume.CreateVolumeMsg +import org.zstack.header.volume.CreateVolumeReply +import org.zstack.header.volume.VolumeType +import org.zstack.storage.volume.VolumeSystemTags +import org.zstack.header.volume.VolumeConstant import org.zstack.kvm.KVMAgentCommands import org.zstack.kvm.KVMConstant import org.zstack.kvm.VolumeTO @@ -24,7 +38,10 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference /** * Covers the ZBS Vhost output-protocol branches of ZbsStorageController that the @@ -43,6 +60,7 @@ import java.util.concurrent.atomic.AtomicBoolean */ class ZbsVhostVolumeCase extends SubCase { EnvSpec env + CloudBus bus PrimaryStorageInventory ps DiskOfferingInventory diskOffering ClusterInventory cluster @@ -143,10 +161,16 @@ class ZbsVhostVolumeCase extends SubCase { instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory image = env.inventoryByName("image") as ImageInventory l3 = env.inventoryByName("l3") as L3NetworkInventory + bus = bean(CloudBus.class) testDefaultOutputProtocolIsVhost() testVhostDataVolumeCreateDeleteLifecycle() + testChangeVolumeProtocol() + testCreateDataVolumeWithExplicitProtocol() + testVolumeProtocolSystemTagConsumedIntoVolume() + testUnknownVolumeProtocolSystemTagRejected() testVhostVmStartActivationChain() + testAddProtocolPreparesHostsAndRecordsProtocolRefs() } } @@ -199,6 +223,141 @@ class ZbsVhostVolumeCase extends SubCase { .isExists() } + // a PS can expose multiple output protocols; APIChangeVolumeProtocolMsg switches + // an idle volume between them offline (persist VolumeVO.protocol; next activate + // builds the new path). validates the target protocol is one the PS exposes. + void testChangeVolumeProtocol() { + // PS starts with Vhost only; add CBD so the volume can switch to it + addStorageProtocol { + uuid = ps.uuid + outputProtocol = VolumeProtocol.CBD.toString() + } + + VolumeInventory vol = createDataVolume { + name = "switch-data" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + } as VolumeInventory + assert vol.protocol == VolumeProtocol.Vhost.toString() + + changeVolumeProtocol { + volumeUuid = vol.uuid + protocol = VolumeProtocol.CBD.toString() + } + def proto = Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, vol.uuid) + .select(VolumeAO_.protocol) + .findValue() + assert proto == VolumeProtocol.CBD.toString() : \ + "volume protocol not switched to CBD: actual=${proto}" + + // switching to a protocol the PS does not expose must be rejected + expect(AssertionError.class) { + changeVolumeProtocol { + volumeUuid = vol.uuid + protocol = VolumeProtocol.NBD.toString() + } + } + + // switching to the protocol it already uses must be rejected + expect(AssertionError.class) { + changeVolumeProtocol { + volumeUuid = vol.uuid + protocol = VolumeProtocol.CBD.toString() + } + } + + deleteDataVolume { uuid = vol.uuid } + expungeDataVolume { uuid = vol.uuid } + } + + // create-time protocol selection: an explicit protocol on the create request + // overrides the PS default (Vhost here) and must be one the PS exposes. mirror of + // APIChangeVolumeProtocolMsg validation so create and change agree on outputProtocols. + void testCreateDataVolumeWithExplicitProtocol() { + // CBD was added to the PS by testChangeVolumeProtocol; default is still Vhost. + // asking for CBD explicitly must win over the Vhost default. + VolumeInventory vol = createDataVolume { + name = "explicit-cbd" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + protocol = VolumeProtocol.CBD.toString() + } as VolumeInventory + assert vol.protocol == VolumeProtocol.CBD.toString() : \ + "explicit create protocol ignored: expected=CBD actual=${vol.protocol}" + + deleteDataVolume { uuid = vol.uuid } + expungeDataVolume { uuid = vol.uuid } + + // a protocol the PS does not expose must be rejected at create, same as change + expect(AssertionError.class) { + createDataVolume { + name = "explicit-nbd" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + protocol = VolumeProtocol.NBD.toString() + } + } + } + + // the Cloud VM-create path provides no DiskAO, so a per-volume protocol can only + // ride in as a volumeProtocol::{protocol} ephemeral system tag. VmAllocateVolumeFlow + // folds those tags into the CreateVolumeMsg of each volume, and createVolume reads the + // token into VolumeVO.protocol; being ephemeral, the framework never persists it as a + // resident tag. drive CreateVolumeMsg directly to prove that convergence point: CBD + // here overrides the PS Vhost default, exactly like an explicit create protocol would. + void testVolumeProtocolSystemTagConsumedIntoVolume() { + String protocolTag = VolumeSystemTags.VOLUME_PROTOCOL.instantiateTag( + [(VolumeSystemTags.VOLUME_PROTOCOL_TOKEN): VolumeProtocol.CBD.toString()]) + + CreateVolumeMsg msg = new CreateVolumeMsg() + msg.setName("systag-cbd") + msg.setSize(SizeUnit.GIGABYTE.toByte(1)) + msg.setFormat("qcow2") + msg.setVolumeType(VolumeType.Data.toString()) + msg.setPrimaryStorageUuid(ps.uuid) + msg.setAccountUuid(AccountConstant.INITIAL_SYSTEM_ADMIN_UUID) + msg.setSystemTags([protocolTag]) + bus.makeLocalServiceId(msg, VolumeConstant.SERVICE_ID) + + CreateVolumeReply reply = syncSend(msg) as CreateVolumeReply + assert reply.isSuccess() : "raw CreateVolumeMsg failed: ${reply.error}" + // reply.inventory is org.zstack.header.volume.VolumeInventory; keep it dynamic + // so the org.zstack.sdk.* star import does not coerce it to the SDK type. + def vol = reply.inventory + assert vol.protocol == VolumeProtocol.CBD.toString() : \ + "systemTag protocol not consumed: expected=CBD actual=${vol.protocol}" + assert !VolumeSystemTags.VOLUME_PROTOCOL.hasTag(vol.uuid) : \ + "volumeProtocol tag must be stripped after consume, still present on ${vol.uuid}" + + // this volume was minted NotInstantiated via a raw CreateVolumeMsg (installPath + // null), so the normal expunge path would call deactivateAndDeleteVolume on the + // external PS with a null installPath. drop the synthetic fixture row directly + // instead of exercising that delete-on-PS flow. + SQL.New(VolumeVO.class).eq(VolumeVO_.uuid, vol.uuid).hardDelete() + } + + // the enum guard runs in VolumeApiInterceptor before any allocation flow, reading + // the protocol tag straight off APICreateVmInstanceMsg.dataVolumeSystemTags. a bad + // protocol token must be rejected at API time, not silently set on the volume. + void testUnknownVolumeProtocolSystemTagRejected() { + expectApiFailure({ + createVmInstance { + name = "bad-protocol-vm" + instanceOfferingUuid = instanceOffering.uuid + imageUuid = image.uuid + l3NetworkUuids = [l3.uuid] + primaryStorageUuidForRootVolume = ps.uuid + dataVolumeSystemTags = [VolumeSystemTags.VOLUME_PROTOCOL.instantiateTag( + [(VolumeSystemTags.VOLUME_PROTOCOL_TOKEN): "BOGUS"])] + } + }) { + // assert it failed on OUR guard, not some unrelated admission error + assert JSONObjectUtil.toJsonString(delegate).contains("unsupported volume protocol") : \ + "rejected for the wrong reason: ${JSONObjectUtil.toJsonString(delegate)}" + } + } + // full VM-start chain. a Vhost root volume forces the framework to activate the // volume on the host before boot, routing through ZbsStorageController: // activate(Vhost) -> KVMHostAsyncHttpCallMsg(VHOST_ACTIVATE_PATH) -> socketPath @@ -241,12 +400,23 @@ class ZbsVhostVolumeCase extends SubCase { "activate cmd controllerName malformed: ${cmd.controllerName}" assert cmd.socketDir == ZbsConstants.VHOST_SOCKET_DIR : \ "activate cmd socketDir expected=${ZbsConstants.VHOST_SOCKET_DIR} actual=${cmd.socketDir}" + // auto-deploy params: the agent lazily ensures the SPDK target from these. + // cores left null lets the agent compute them; coreCount carries the default. + assert cmd.image != null : "activate cmd missing target image for lazy deploy" + assert cmd.coreCount != null && cmd.coreCount > 0 : \ + "activate cmd missing coreCount for agent-computed cores: ${cmd.coreCount}" activateCalled.set(true) def rsp = new ZbsStorageController.VhostActivateRsp() rsp.socketPath = cmd.socketDir + "/" + cmd.controllerName return rsp } + // vm destroy deactivates the vhost volume on the host; without a handler the + // 404 makes the destroy teardown flaky + env.simulator(ZbsStorageController.VHOST_DEACTIVATE_PATH) { HttpEntity e, EnvSpec spec -> + return new ZbsStorageController.VhostActivateRsp() + } + // end of the chain: the VM boots with a vhost-user-blk root disk whose path is // the SPDK unix socket. assert the TO the framework built from getActivePath. env.afterSimulator(KVMConstant.KVM_START_VM_PATH) { rsp, HttpEntity e -> @@ -296,4 +466,153 @@ class ZbsVhostVolumeCase extends SubCase { clusterUuid = cluster.uuid } } + + // adding an output protocol on a PS with connected hosts must prepare every + // host for the protocol (Vhost -> deploy the SPDK target over + // VHOST_TARGET_ENSURE_PATH) and record per-protocol connectivity rows that the + // frontend reads through QueryExternalPrimaryStorageHostProtocolRef. the + // host-level ref row keeps the legacy folded all-protocol semantics. + void testAddProtocolPreparesHostsAndRecordsProtocolRefs() { + HostInventory host = env.inventoryByName("kvm-1") as HostInventory + + AtomicBoolean ensureCalled = new AtomicBoolean(false) + AtomicBoolean vhostTargetHealthy = new AtomicBoolean(true) + + env.simulator(ZbsStorageController.VHOST_TARGET_ENSURE_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.VhostActivateCmd.class) + assert cmd.image != null : "ensure cmd missing target image for lazy deploy" + ensureCalled.set(true) + return new ZbsStorageController.CheckHostStorageConnectionRsp() + } + env.simulator(ZbsStorageController.VHOST_TARGET_HEALTH_PATH) { HttpEntity e, EnvSpec spec -> + def rsp = new ZbsStorageController.VhostTargetHealthRsp() + rsp.healthy = vhostTargetHealthy.get() + return rsp + } + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + def vrsp = new ZbsStorageController.CreateVolumeRsp() + vrsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}".toString() + return vrsp + } + return rsp + } + + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + + // mimic a storage from before vhost support: drop the Vhost protocol row, + // then add it back through the API, which must prepare the connected hosts + SQL.New(PrimaryStorageOutputProtocolRefVO.class) + .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(PrimaryStorageOutputProtocolRefVO_.outputProtocol, VolumeProtocol.Vhost.toString()) + .delete() + + addStorageProtocol { + uuid = ps.uuid + outputProtocol = VolumeProtocol.Vhost.toString() + } + + assert ensureCalled.get() : \ + "addStorageProtocol(Vhost) did not reach VHOST_TARGET_ENSURE_PATH on the connected host" + + // the protocol row write is fire-and-forget behind the PS queue + retryInSecs { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() + } + + // frontend contract: per-protocol connectivity is queryable + def refs = queryExternalPrimaryStorageHostProtocolRef { + conditions = ["primaryStorageUuid=${ps.uuid}".toString()] + } as List + assert refs.find { it.protocol == VolumeProtocol.Vhost.toString() && it.hostUuid == host.uuid } != null : \ + "query api returned no Vhost connectivity row: ${refs}" + + // periodic pings drive the per-protocol health reports; shorten the + // interval so the status flips below land within the retry windows + updateGlobalConfig { + category = "host" + name = "ping.interval" + value = 1 + } + + // every reported protocol gets its own connectivity row on ping + retryInSecs(30) { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.CBD.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() + } + + // a dead vhost target flips its own protocol row while the CBD row keeps + // its own state; the host-level row folds all protocols (legacy + // semantics) so it goes Disconnected too + vhostTargetHealthy.set(false) + retryInSecs(30) { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Disconnected) + .isExists() + } + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.CBD.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() : "the CBD row must keep its own state independent of the vhost target" + retryInSecs(30) { + assert Q.New(PrimaryStorageHostRefVO.class) + .eq(PrimaryStorageHostRefVO_.primaryStorageUuid, ps.uuid) + .eq(PrimaryStorageHostRefVO_.hostUuid, host.uuid) + .eq(PrimaryStorageHostRefVO_.status, PrimaryStorageHostStatus.Disconnected) + .isExists() + } + + // the target recovers: rows self-heal on the next report + vhostTargetHealthy.set(true) + retryInSecs(30) { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() + assert Q.New(PrimaryStorageHostRefVO.class) + .eq(PrimaryStorageHostRefVO_.primaryStorageUuid, ps.uuid) + .eq(PrimaryStorageHostRefVO_.hostUuid, host.uuid) + .eq(PrimaryStorageHostRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() + } + + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + } + + private MessageReply syncSend(org.zstack.header.message.Message msg) { + AtomicReference ref = new AtomicReference<>() + CountDownLatch done = new CountDownLatch(1) + bus.send(msg, new CloudBusCallBack(null) { + @Override + void run(MessageReply reply) { + ref.set(reply) + done.countDown() + } + }) + assert done.await(30, TimeUnit.SECONDS) : "timed out waiting for ${msg.class.simpleName} reply" + return ref.get() + } } diff --git a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy index a09908b89ca..2c63153c571 100644 --- a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy +++ b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy @@ -3689,6 +3689,33 @@ abstract class ApiHelper { } + def attachDGpuToVm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.AttachDGpuToVmAction.class) Closure c) { + def a = new org.zstack.sdk.AttachDGpuToVmAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def attachDataVolumeToHost(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.AttachDataVolumeToHostAction.class) Closure c) { def a = new org.zstack.sdk.AttachDataVolumeToHostAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -6848,6 +6875,33 @@ abstract class ApiHelper { } + def changeVolumeProtocol(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.ChangeVolumeProtocolAction.class) Closure c) { + def a = new org.zstack.sdk.ChangeVolumeProtocolAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def changeVolumeState(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.ChangeVolumeStateAction.class) Closure c) { def a = new org.zstack.sdk.ChangeVolumeStateAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -18620,33 +18674,6 @@ abstract class ApiHelper { } - def attachDGpuToVm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.AttachDGpuToVmAction.class) Closure c) { - def a = new org.zstack.sdk.AttachDGpuToVmAction() - a.sessionId = Test.currentEnvSpec?.session?.uuid - c.resolveStrategy = Closure.OWNER_FIRST - c.delegate = a - c() - - - if (System.getProperty("apipath") != null) { - if (a.apiId == null) { - a.apiId = Platform.uuid - } - - def tracker = new ApiPathTracker(a.apiId) - def out = errorOut(a.call()) - def path = tracker.getApiPath() - if (!path.isEmpty()) { - Test.apiPaths[a.class.name] = path.join(" --->\n") - } - - return out - } else { - return errorOut(a.call()) - } - } - - def detachBaremetalPxeServerFromCluster(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.DetachBaremetalPxeServerFromClusterAction.class) Closure c) { def a = new org.zstack.sdk.DetachBaremetalPxeServerFromClusterAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -31351,6 +31378,35 @@ abstract class ApiHelper { } + def queryExternalPrimaryStorageHostProtocolRef(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefAction.class) Closure c) { + def a = new org.zstack.sdk.QueryExternalPrimaryStorageHostProtocolRefAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def queryExternalServiceConfiguration(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryExternalServiceConfigurationAction.class) Closure c) { def a = new org.zstack.sdk.QueryExternalServiceConfigurationAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -40683,33 +40739,6 @@ abstract class ApiHelper { } - def updateVmDGpu(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdateVmDGpuAction.class) Closure c) { - def a = new org.zstack.sdk.UpdateVmDGpuAction() - a.sessionId = Test.currentEnvSpec?.session?.uuid - c.resolveStrategy = Closure.OWNER_FIRST - c.delegate = a - c() - - - if (System.getProperty("apipath") != null) { - if (a.apiId == null) { - a.apiId = Platform.uuid - } - - def tracker = new ApiPathTracker(a.apiId) - def out = errorOut(a.call()) - def path = tracker.getApiPath() - if (!path.isEmpty()) { - Test.apiPaths[a.class.name] = path.join(" --->\n") - } - - return out - } else { - return errorOut(a.call()) - } - } - - def setVmEmulatorPinning(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.SetVmEmulatorPinningAction.class) Closure c) { def a = new org.zstack.sdk.SetVmEmulatorPinningAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -48108,6 +48137,33 @@ abstract class ApiHelper { } + def updateVmDGpu(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdateVmDGpuAction.class) Closure c) { + def a = new org.zstack.sdk.UpdateVmDGpuAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def updateVmInstance(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdateVmInstanceAction.class) Closure c) { def a = new org.zstack.sdk.UpdateVmInstanceAction() a.sessionId = Test.currentEnvSpec?.session?.uuid