From 7c23b2610a83bd09af6f2f200a36de791e30b6d8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 13 Jan 2026 16:15:33 +0530 Subject: [PATCH 001/129] [WIP] plugins: veeam control service Signed-off-by: Abhishek Kumar --- client/pom.xml | 5 + .../veeam-control-service/pom.xml | 61 ++++ .../apache/cloudstack/veeam/RouteHandler.java | 34 +++ .../cloudstack/veeam/VeeamControlServer.java | 83 ++++++ .../cloudstack/veeam/VeeamControlService.java | 38 +++ .../veeam/VeeamControlServiceImpl.java | 82 ++++++ .../cloudstack/veeam/VeeamControlServlet.java | 126 ++++++++ .../cloudstack/veeam/api/ApiService.java | 44 +++ .../cloudstack/veeam/api/VmsRouteHandler.java | 184 ++++++++++++ .../converter/UserVmJoinVOToVmConverter.java | 124 ++++++++ .../cloudstack/veeam/api/dto/ActionLink.java | 39 +++ .../cloudstack/veeam/api/dto/Actions.java | 33 +++ .../apache/cloudstack/veeam/api/dto/Bios.java | 31 ++ .../apache/cloudstack/veeam/api/dto/Cpu.java | 33 +++ .../cloudstack/veeam/api/dto/Fault.java | 35 +++ .../apache/cloudstack/veeam/api/dto/Link.java | 33 +++ .../apache/cloudstack/veeam/api/dto/Os.java | 31 ++ .../apache/cloudstack/veeam/api/dto/Ref.java | 39 +++ .../cloudstack/veeam/api/dto/Topology.java | 35 +++ .../apache/cloudstack/veeam/api/dto/Vm.java | 69 +++++ .../veeam/api/request/VmListQuery.java | 106 +++++++ .../veeam/api/request/VmSearchExpr.java | 103 +++++++ .../veeam/api/request/VmSearchFilters.java | 62 ++++ .../veeam/api/request/VmSearchParser.java | 274 ++++++++++++++++++ .../veeam/api/response/FaultResponse.java | 39 +++ .../api/response/VmCollectionResponse.java | 47 +++ .../veeam/api/response/VmEntityResponse.java | 34 +++ .../veeam/filter/BasicAuthFilter.java | 110 +++++++ .../cloudstack/veeam/sso/SsoService.java | 72 +++++ .../cloudstack/veeam/utils/Negotiation.java | 45 +++ .../veeam/utils/ResponseMapper.java | 59 ++++ .../veeam/utils/ResponseWriter.java | 80 +++++ .../veeam-control-service/module.properties | 18 ++ .../spring-veeam-control-service-context.xml | 41 +++ plugins/pom.xml | 1 + 35 files changed, 2250 insertions(+) create mode 100644 plugins/integrations/veeam-control-service/pom.xml create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BasicAuthFilter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Negotiation.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties create mode 100644 plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml diff --git a/client/pom.xml b/client/pom.xml index b8dffe65d4fb..88a32c6f6461 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -612,6 +612,11 @@ cloud-plugin-backup-nas ${project.version} + + org.apache.cloudstack + cloud-plugin-integrations-veeam-control-service + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service diff --git a/plugins/integrations/veeam-control-service/pom.xml b/plugins/integrations/veeam-control-service/pom.xml new file mode 100644 index 000000000000..cc0349b75d60 --- /dev/null +++ b/plugins/integrations/veeam-control-service/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + cloud-plugin-integrations-veeam-control-service + Apache CloudStack Plugin - Veeam Control Service + + org.apache.cloudstack + cloudstack-plugins + 4.22.1.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-utils + ${project.version} + + + org.apache.cloudstack + cloud-api + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + ${cs.jetty.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${cs.jackson.version} + + + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java new file mode 100644 index 000000000000..25c4dfbf8f66 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.utils.Negotiation; + +import com.cloud.utils.component.Adapter; + +public interface RouteHandler extends Adapter { + default int priority() { return 0; } + boolean canHandle(String method, String path) throws IOException; + void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) + throws IOException; +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java new file mode 100644 index 000000000000..aa03cddd2f72 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +import javax.servlet.DispatcherType; + +import org.apache.cloudstack.veeam.filter.BasicAuthFilter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +public class VeeamControlServer { + private static final Logger LOGGER = LogManager.getLogger(VeeamControlServer.class); + + private Server server; + private List routeHandlers; + + public VeeamControlServer(List routeHandlers) { + this.routeHandlers = new ArrayList<>(routeHandlers); + this.routeHandlers.sort((a, b) -> Integer.compare(b.priority(), a.priority())); + } + + public void startIfEnabled() throws Exception { + final boolean enabled = VeeamControlService.Enabled.value(); + if (!enabled) { + LOGGER.info("Veeam Control API server is disabled"); + return; + } + + final String bind = VeeamControlService.BindAddress.value(); + final int port = VeeamControlService.Port.value(); + String ctxPath = VeeamControlService.ContextPath.value(); + LOGGER.info("Veeam Control server - bind: {}, port: {}, context: {} with {} handlers", bind, port, ctxPath, + routeHandlers != null ? routeHandlers.size() : 0); + + + server = new Server(new InetSocketAddress(bind, port)); + + final ServletContextHandler ctx = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + ctx.setContextPath(ctxPath); + + // Basic Auth for all routes + ctx.addFilter(BasicAuthFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST)); + + // Front controller servlet + ctx.addServlet(new ServletHolder(new VeeamControlServlet(routeHandlers)), "/*"); + + server.setHandler(ctx); + server.start(); + + LOGGER.info("Started Veeam Control API server on {}:{} with context {}", bind, port, ctxPath); + } + + public void stop() throws Exception { + if (server != null) { + server.stop(); + server = null; + } + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java new file mode 100644 index 000000000000..87233a3ebd8c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.utils.component.PluggableService; + +public interface VeeamControlService extends PluggableService, Configurable { + ConfigKey Enabled = new ConfigKey<>("Advanced", Boolean.class, "integration.veeam.control.enabled", + "false", "Enable the Veeam Integration REST API server", false); + ConfigKey BindAddress = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.bind.address", + "127.0.0.1", "Bind address for Veeam Integration REST API server", false); + ConfigKey Port = new ConfigKey<>("Advanced", Integer.class, "integration.veeam.control.port", + "8090", "Port for Veeam Integration REST API server", false); + ConfigKey ContextPath = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.context.path", + "/integrations/veeam", "Context path for Veeam Integration REST API server", false); + ConfigKey Username = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.username", + "veeam", "Username for Basic Auth on Veeam Integration REST API server", true); + ConfigKey Password = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.password", + "change-me", "Password for Basic Auth on Veeam Integration REST API server", true); +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java new file mode 100644 index 000000000000..12e6b58b1ffd --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam; + +import java.util.List; + +import org.apache.cloudstack.framework.config.ConfigKey; + +import com.cloud.utils.component.ManagerBase; + +public class VeeamControlServiceImpl extends ManagerBase implements VeeamControlService { + + private List routeHandlers; + + private VeeamControlServer veeamControlServer; + + public List getRouteHandlers() { + return routeHandlers; + } + + public void setRouteHandlers(final List routeHandlers) { + this.routeHandlers = routeHandlers; + } + + @Override + public boolean start() { + veeamControlServer = new VeeamControlServer(getRouteHandlers()); + try { + veeamControlServer.startIfEnabled(); + } catch (Exception e) { + logger.error("Failed to start Veeam Control API server, continuing without it", e); + } + return true; + } + + @Override + public boolean stop() { + try { + veeamControlServer.stop(); + } catch (Exception e) { + logger.error("Failed to stop Veeam Control API server cleanly", e); + } + return true; + } + + @Override + public List> getCommands() { + return List.of(); + } + + @Override + public String getConfigComponentName() { + return VeeamControlService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + Enabled, + BindAddress, + Port, + ContextPath, + Username, + Password + }; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java new file mode 100644 index 000000000000..bf064e27f025 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam; + + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.ResponseMapper; +import org.apache.cloudstack.veeam.utils.ResponseWriter; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class VeeamControlServlet extends HttpServlet { + private static final Logger LOGGER = LogManager.getLogger(VeeamControlServlet.class); + + private final ResponseWriter writer; + private final List routeHandlers; + + public VeeamControlServlet(List routeHandlers) { + this.routeHandlers = routeHandlers; + ResponseMapper mapper = new ResponseMapper(); + writer = new ResponseWriter(mapper); + } + + public ResponseWriter getWriter() { + return writer; + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { + String method = req.getMethod(); + String path = normalize(req.getPathInfo()); + Negotiation.OutFormat outFormat = Negotiation.responseFormat(req); + + LOGGER.info("Received {} request for {} with out format: {}", method, path, outFormat); + + + try { + if ("/".equals(path)) { + handleRoot(req, resp, outFormat); + return; + } + + if (CollectionUtils.isNotEmpty(this.routeHandlers)) { + for (RouteHandler handler : this.routeHandlers) { + if (handler.canHandle(method, path)) { + handler.handle(req, resp, path, outFormat, this); + return; + } + } + } + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } catch (Error e) { + writer.writeFault(resp, e.status, e.message, null, outFormat); + } + } + + private String normalize(String pathInfo) { + if (pathInfo == null || pathInfo.isBlank()) return "/"; + return pathInfo; + } + + protected void handleRoot(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat) + throws IOException { + + String method = req.getMethod(); + if (!"GET".equals(method) && !"POST".equals(method)) { + // You didn’t list 405; keep it simple with 400 + throw Error.badRequest("Unsupported method for root: " + method); + } + + writer.write(resp, 200, Map.of( + "name", "CloudStack Veeam Control Service", + "pluginVersion", "0.1"), outFormat); + } + + public void methodNotAllowed(final HttpServletResponse resp, final String allow, final Negotiation.OutFormat outFormat) throws IOException { + resp.setHeader("Allow", allow); + writer.writeFault(resp, 405, "Method Not Allowed", "Allowed methods: " + allow, outFormat); + } + + public void badRequest(final HttpServletResponse resp, String detail, Negotiation.OutFormat outFormat) throws IOException { + writer.writeFault(resp, 400, "Bad request", detail, outFormat); + } + + + public void notFound(final HttpServletResponse resp, String detail, Negotiation.OutFormat outFormat) throws IOException { + writer.writeFault(resp, 404, "Not found", detail, outFormat); + } + + public static class Error extends RuntimeException { + final int status; + final String message; + public Error(int status, String message) { + super(message); + this.status = status; + this.message = message; + } + public static Error badRequest(String msg) { return new Error(400, msg); } + public static Error unauthorized(String msg) { return new Error(401, msg); } + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java new file mode 100644 index 000000000000..d2a9cf386f01 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.utils.Negotiation; + +import com.cloud.utils.component.ManagerBase; + +public class ApiService extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api"; + + @Override + public boolean canHandle(String method, String path) { + return path.startsWith("/api"); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + // ToDo: handle root API requests + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", null, outFormat); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java new file mode 100644 index 000000000000..166794e37bb0 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; +import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.cloudstack.veeam.api.request.VmListQuery; +import org.apache.cloudstack.veeam.api.request.VmSearchExpr; +import org.apache.cloudstack.veeam.api.request.VmSearchFilters; +import org.apache.cloudstack.veeam.api.request.VmSearchParser; +import org.apache.cloudstack.veeam.api.response.VmCollectionResponse; +import org.apache.cloudstack.veeam.api.response.VmEntityResponse; +import org.apache.cloudstack.veeam.utils.Negotiation; + +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.utils.component.ManagerBase; + +public class VmsRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/vms"; + private static final int DEFAULT_MAX = 50; + private static final int HARD_CAP_MAX = 1000; + private static final int DEFAULT_PAGE = 1; + + @Inject + UserVmJoinDao userVmJoinDao; + + private VmSearchParser searchParser; + + @Override + public boolean start() { + + this.searchParser = new VmSearchParser(Set.of( + "id", "name", "status", "cluster", "host", "template" + )); + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return path.startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (path.equals(BASE_ROUTE)) { + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + handleGet(req, resp, outFormat, io); + return; + } + + // /api/vms/{id} + final String vmId = matchSinglePathParam(path, BASE_ROUTE + "/"); + if (vmId != null) { + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + handleGetById(vmId, resp, outFormat, io); + return; + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + /** + * Matches /api/vms/{id} where {id} is a single path segment (no extra '/'). + * Returns id or null. + */ + private static String matchSinglePathParam(final String path, final String prefix) { + if (!path.startsWith(prefix)) return null; + final String rest = path.substring(prefix.length()); // after "/api/vms/" + if (rest.isEmpty()) return null; + if (rest.contains("/")) return null; // ensure only 1 segment + return rest; + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final VmListQuery q = fromRequest(req); + + // Validate max/page early (optional strictness) + if (q.getMax() != null && q.getMax() <= 0) { + io.notFound(resp, "Invalid 'max' (must be > 0)", outFormat); + return; + } + if (q.getPage() != null && q.getPage() <= 0) { + io.notFound(resp, "Invalid 'page' (must be > 0)", outFormat); + return; + } + + final int limit = q.resolvedMax(DEFAULT_MAX, HARD_CAP_MAX); + final int offset = q.offset(DEFAULT_MAX, HARD_CAP_MAX, DEFAULT_PAGE); + + final VmSearchExpr expr; + try { + expr = searchParser.parse(q.getSearch()); + } catch (VmSearchParser.VmSearchParseException e) { + io.notFound(resp, "Invalid search: " + e.getMessage(), outFormat); + return; + } + + final VmSearchFilters filters; + try { + filters = VmSearchFilters.fromAndOnly(expr); // AND-only v1 + } catch (VmSearchParser.VmSearchParseException e) { + io.notFound(resp, "Unsupported search: " + e.getMessage(), outFormat); + return; + } + + final List result = UserVmJoinVOToVmConverter.toVmList(listUserVms()); + final VmCollectionResponse response = new VmCollectionResponse(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + private static VmListQuery fromRequest(final HttpServletRequest req) { + final VmListQuery q = new VmListQuery(); + q.setSearch(req.getParameter("search")); + q.setMax(parseIntOrNull(req.getParameter("max"))); + q.setPage(parseIntOrNull(req.getParameter("page"))); + return q; + } + + private static Integer parseIntOrNull(final String s) { + if (s == null || s.trim().isEmpty()) return null; + try { + return Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + return Integer.valueOf(-1); // will be rejected by validation above + } + } + + protected List listUserVms() { + // Todo: add filtering, pagination + return userVmJoinDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); + if (userVmJoinVO == null) { + io.notFound(resp, "VM not found: " + id, outFormat); + return; + } + VmEntityResponse response = new VmEntityResponse(UserVmJoinVOToVmConverter.toVm(userVmJoinVO)); + + io.getWriter().write(resp, 200, response, outFormat); + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java new file mode 100644 index 000000000000..ba5c831169d0 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Cpu; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Topology; +import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.vm.VirtualMachine; + +public final class UserVmJoinVOToVmConverter { + + private UserVmJoinVOToVmConverter() { + } + + /** + * Convert CloudStack UserVmJoinVO -> oVirt-like Vm DTO. + * + * @param src UserVmJoinVO + */ + public static Vm toVm(final UserVmJoinVO src) { + if (src == null) { + return null; + } + final String basePath = VeeamControlService.ContextPath.value(); + final Vm dst = new Vm(); + + dst.id = src.getUuid(); + dst.name = StringUtils.firstNonBlank(src.getName(), src.getInstanceName()); + // CloudStack doesn't really have "description" for VM; displayName is closest + dst.description = src.getDisplayName(); + dst.href = basePath + VmsRouteHandler.BASE_ROUTE + "/" + src.getUuid(); + dst.status = mapStatus(src.getState()); + final Date lastUpdated = src.getLastUpdated(); + if ("down".equals(dst.status)) { + dst.stopTime = lastUpdated.getTime(); + } + final Ref template = buildRef( + basePath + ApiService.BASE_ROUTE, + "template", + src.getTemplateUuid() + ); + dst.template = template; + dst.originalTemplate = template; + dst.host = buildRef( + basePath + ApiService.BASE_ROUTE, + "host", + src.getHostUuid()); + dst.cluster = buildRef( + basePath + ApiService.BASE_ROUTE, + "cluster", + src.getHostUuid()); + dst.memory = (long) src.getRamSize(); + + dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); + dst.os = null; + dst.bios = null; + dst.actions = null; + dst.link = null; + + return dst; + } + + public static List toVmList(final List srcList) { + return srcList.stream() + .map(UserVmJoinVOToVmConverter::toVm) + .collect(Collectors.toList()); + } + + private static String mapStatus(final VirtualMachine.State state) { + if (state == null) { + return null; + } + + // CloudStack-ish states -> oVirt-ish up/down + if (Arrays.asList(VirtualMachine.State.Running, VirtualMachine.State.Starting, + VirtualMachine.State.Migrating, VirtualMachine.State.Restoring).contains(state)) { + return "up"; + } + if (Arrays.asList(VirtualMachine.State.Stopped, VirtualMachine.State.Stopping, + VirtualMachine.State.Shutdown, VirtualMachine.State.Error, + VirtualMachine.State.Expunging).contains(state)) { + return "down"; + } + return null; + } + + private static Ref buildRef(final String baseHref, final String suffix, final String id) { + if (StringUtils.isBlank(id)) { + return null; + } + final Ref r = new Ref(); + r.id = id; + r.href = (baseHref != null) ? (baseHref + "/" + suffix + "/" + id) : null; + return r; + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java new file mode 100644 index 000000000000..fe127d63364c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class ActionLink { + public String rel; // start/stop/reboot/shutdown... + public String href; // /api/vms/{id}/start + public String method; // "post" + + public ActionLink() {} + + public ActionLink(final String rel, final String href, final String method) { + this.rel = rel; + this.href = href; + this.method = method; + } + + public static ActionLink post(final String rel, final String href) { + return new ActionLink(rel, href, "post"); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java new file mode 100644 index 000000000000..a834c579973e --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Actions { + public List link; + + public Actions() {} + + public Actions(final List link) { + this.link = link; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java new file mode 100644 index 000000000000..f1de8cf3a5ad --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Bios { + public String type; // "uefi" or "bios" or whatever mapping you choose + + public Bios() {} + + public Bios(final String type) { + this.type = type; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java new file mode 100644 index 000000000000..bc3859d89980 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Cpu { + public String architecture; + public Topology topology; + + public Cpu() {} + + public Cpu(final String architecture, final Topology topology) { + this.architecture = architecture; + this.topology = topology; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java new file mode 100644 index 000000000000..51d4e6eca576 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "fault") +public final class Fault { + public String reason; // "Not Found", "Bad Request", "Unauthorized" + public String detail; // full message + + public Fault() {} + + public Fault(final String reason, final String detail) { + this.reason = reason; + this.detail = detail; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java new file mode 100644 index 000000000000..276cd0a6a5cb --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Link { + public String rel; + public String href; + + public Link() {} + + public Link(final String rel, final String href) { + this.rel = rel; + this.href = href; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java new file mode 100644 index 000000000000..e53374e4d103 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Os { + public String type; // "rhel_9", "windows_2022", etc. + + public Os() {} + + public Os(final String type) { + this.type = type; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java new file mode 100644 index 000000000000..04ab01f6abdb --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Ref { + public String href; + public String id; + public String name; // optional + + public Ref() {} + + public Ref(final String href, final String id, final String name) { + this.href = href; + this.id = id; + this.name = name; + } + + public static Ref of(final String href, final String id) { + return new Ref(href, id, null); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java new file mode 100644 index 000000000000..3458b2cb17f2 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Topology { + public Integer sockets; + public Integer cores; + public Integer threads; + + public Topology() {} + + public Topology(final Integer sockets, final Integer cores, final Integer threads) { + this.sockets = sockets; + this.cores = cores; + this.threads = threads; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java new file mode 100644 index 000000000000..ce1b64e53039 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +/** + * VM DTO intentionally uses snake_case field names to match the required JSON. + * Configure Jackson globally with SNAKE_CASE or keep as-is. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "vm") +public final class Vm { + public String href; + public String id; + public String name; + public String description; + + public String status; // "up", "down", ... + + @JsonProperty("stop_reason") + @JacksonXmlProperty(localName = "stop_reason") + public String stopReason; // empty string allowed + + @JsonProperty("stop_time") + @JacksonXmlProperty(localName = "stop_time") + public Long stopTime; // epoch millis + + public Ref template; + + @JsonProperty("original_template") + @JacksonXmlProperty(localName = "original_template") + public Ref originalTemplate; + + public Ref cluster; + public Ref host; + + public Long memory; // bytes + public Cpu cpu; + public Os os; + public Bios bios; + + public Actions actions; // actions.link[] + @JacksonXmlElementWrapper(useWrapping = false) + public List link; // related resources + + public Vm() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java new file mode 100644 index 000000000000..9383979c2b72 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.request; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Query parameters supported by GET /api/vms (oVirt-like). + * + * Examples: + * /api/vms?search=name=myvm&max=50&page=1 + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class VmListQuery { + + /** + * oVirt-like search expression, e.g.: + * name=myvm + * status=down + * name=myvm and status=up + */ + @JsonProperty("search") + private String search; + + /** + * Max number of entries to return. + */ + @JsonProperty("max") + private Integer max; + + /** + * 1-based page number. + */ + @JsonProperty("page") + private Integer page; + + public VmListQuery() { + } + + public VmListQuery(final String search, final Integer max, final Integer page) { + this.search = search; + this.max = max; + this.page = page; + } + + public String getSearch() { + return search; + } + + public void setSearch(final String search) { + this.search = search; + } + + public Integer getMax() { + return max; + } + + public void setMax(final Integer max) { + this.max = max; + } + + public Integer getPage() { + return page; + } + + public void setPage(final Integer page) { + this.page = page; + } + + // ----- helpers (optional, but convenient) ----- + + @JsonIgnore + public int resolvedMax(final int defaultMax, final int hardCap) { + final int m = (max == null || max <= 0) ? defaultMax : max; + return Math.min(m, hardCap); + } + + @JsonIgnore + public int resolvedPage(final int defaultPage) { + return (page == null || page <= 0) ? defaultPage : page; + } + + @JsonIgnore + public int offset(final int defaultMax, final int hardCap, final int defaultPage) { + final int p = resolvedPage(defaultPage); + final int m = resolvedMax(defaultMax, hardCap); + return (p - 1) * m; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java new file mode 100644 index 000000000000..56f8a38e4892 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.request; + +import java.util.Objects; + +/** + * Small AST for oVirt-like search. + * + * Supported grammar: + * expr := orExpr + * orExpr := andExpr (OR andExpr)* + * andExpr := primary (AND primary)* + * primary := '(' expr ')' | term + * term := IDENT '=' (IDENT | STRING) + */ +public interface VmSearchExpr { + + final class Term implements VmSearchExpr { + private final String field; + private final String value; + + public Term(final String field, final String value) { + this.field = Objects.requireNonNull(field, "field"); + this.value = Objects.requireNonNull(value, "value"); + } + + public String getField() { + return field; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "Term(" + field + "=" + value + ")"; + } + } + + final class And implements VmSearchExpr { + private final VmSearchExpr left; + private final VmSearchExpr right; + + public And(final VmSearchExpr left, final VmSearchExpr right) { + this.left = Objects.requireNonNull(left, "left"); + this.right = Objects.requireNonNull(right, "right"); + } + + public VmSearchExpr getLeft() { + return left; + } + + public VmSearchExpr getRight() { + return right; + } + + @Override + public String toString() { + return "And(" + left + ", " + right + ")"; + } + } + + final class Or implements VmSearchExpr { + private final VmSearchExpr left; + private final VmSearchExpr right; + + public Or(final VmSearchExpr left, final VmSearchExpr right) { + this.left = Objects.requireNonNull(left, "left"); + this.right = Objects.requireNonNull(right, "right"); + } + + public VmSearchExpr getLeft() { + return left; + } + + public VmSearchExpr getRight() { + return right; + } + + @Override + public String toString() { + return "Or(" + left + ", " + right + ")"; + } + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java new file mode 100644 index 000000000000..7cf12c0e32c9 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.request; + +import java.util.LinkedHashMap; +import java.util.Map; + +public final class VmSearchFilters { + + private final Map equals = new LinkedHashMap<>(); + + public Map equals() { + return equals; + } + + public VmSearchFilters put(final String field, final String value) { + equals.put(field, value); + return this; + } + + public static VmSearchFilters fromAndOnly(final VmSearchExpr expr) { + final VmSearchFilters f = new VmSearchFilters(); + if (expr == null) { + return f; + } + collect(expr, f); + return f; + } + + private static void collect(final VmSearchExpr expr, final VmSearchFilters f) { + if (expr instanceof VmSearchExpr.Term) { + final VmSearchExpr.Term t = (VmSearchExpr.Term) expr; + f.put(t.getField(), t.getValue()); + return; + } + if (expr instanceof VmSearchExpr.And) { + final VmSearchExpr.And a = (VmSearchExpr.And) expr; + collect(a.getLeft(), f); + collect(a.getRight(), f); + return; + } + if (expr instanceof VmSearchExpr.Or) { + throw new VmSearchParser.VmSearchParseException("Only AND expressions are supported currently"); + } + throw new VmSearchParser.VmSearchParseException("Unsupported search expression: " + expr.getClass().getName()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java new file mode 100644 index 000000000000..e8575750db48 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.request; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Parser for an oVirt-like 'search' parameter. + * + * Examples: + * name=myvm + * status=down and cluster=Default + * name="My VM" or name="Other VM" + * (status=up and host=hv1) or (status=down and host=hv2) + * + * Values can be IDENT (unquoted) or STRING (quoted with " ... "). + */ +public final class VmSearchParser { + + public static final class VmSearchParseException extends RuntimeException { + public VmSearchParseException(final String message) { super(message); } + } + + private final Set allowedFields; + + public VmSearchParser(final Set allowedFields) { + this.allowedFields = allowedFields; + } + + /** + * @return AST or null if input is null/blank + */ + public VmSearchExpr parse(final String input) { + if (input == null || input.trim().isEmpty()) { + return null; + } + final Lexer lexer = new Lexer(input); + final List tokens = lexer.lex(); + final Parser p = new Parser(tokens, allowedFields); + final VmSearchExpr expr = p.parseExpression(); + p.expect(TokenType.EOF); + return expr; + } + + // -------------------- lexer -------------------- + + enum TokenType { + IDENT, STRING, EQ, AND, OR, LPAREN, RPAREN, EOF + } + + static final class Token { + private final TokenType type; + private final String text; + private final int pos; + + Token(final TokenType type, final String text, final int pos) { + this.type = type; + this.text = text; + this.pos = pos; + } + + TokenType type() { return type; } + String text() { return text; } + int pos() { return pos; } + } + + static final class Lexer { + private final String s; + private final int n; + private int i = 0; + + Lexer(final String s) { + this.s = s; + this.n = s.length(); + } + + List lex() { + final List out = new ArrayList<>(); + while (true) { + skipWs(); + if (i >= n) { + out.add(new Token(TokenType.EOF, "", i)); + return out; + } + final char c = s.charAt(i); + + if (c == '(') { + out.add(new Token(TokenType.LPAREN, "(", i++)); + } else if (c == ')') { + out.add(new Token(TokenType.RPAREN, ")", i++)); + } else if (c == '=') { + out.add(new Token(TokenType.EQ, "=", i++)); + } else if (c == '"') { + out.add(readQuoted()); + } else if (isIdentStart(c)) { + out.add(readIdentOrKeyword()); + } else { + throw new VmSearchParseException("Unexpected character '" + c + "' at position " + i); + } + } + } + + private void skipWs() { + while (i < n) { + final char c = s.charAt(i); + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') i++; + else break; + } + } + + private Token readQuoted() { + final int start = i; + i++; // skip opening " + final StringBuilder b = new StringBuilder(); + while (i < n) { + final char c = s.charAt(i); + if (c == '"') { + i++; // closing " + return new Token(TokenType.STRING, b.toString(), start); + } + if (c == '\\') { + if (i + 1 >= n) { + throw new VmSearchParseException("Unterminated escape at position " + i); + } + final char nxt = s.charAt(i + 1); + switch (nxt) { + case '"': b.append('"'); i += 2; break; + case '\\': b.append('\\'); i += 2; break; + case 'n': b.append('\n'); i += 2; break; + case 't': b.append('\t'); i += 2; break; + default: + throw new VmSearchParseException("Unsupported escape \\" + nxt + " at position " + i); + } + continue; + } + b.append(c); + i++; + } + throw new VmSearchParseException("Unterminated string starting at position " + start); + } + + private Token readIdentOrKeyword() { + final int start = i; + i++; + while (i < n && isIdentPart(s.charAt(i))) i++; + + final String text = s.substring(start, i); + final String lower = text.toLowerCase(Locale.ROOT); + + if ("and".equals(lower)) return new Token(TokenType.AND, text, start); + if ("or".equals(lower)) return new Token(TokenType.OR, text, start); + + return new Token(TokenType.IDENT, text, start); + } + + private static boolean isIdentStart(final char c) { + return Character.isLetter(c) || c == '_' || c == '.'; + } + + private static boolean isIdentPart(final char c) { + return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == '-'; + } + } + + // -------------------- parser -------------------- + + static final class Parser { + private final List tokens; + private final Set allowedFields; + private int k = 0; + + Parser(final List tokens, final Set allowedFields) { + this.tokens = tokens; + this.allowedFields = allowedFields; + } + + VmSearchExpr parseExpression() { + return parseOr(); + } + + private VmSearchExpr parseOr() { + VmSearchExpr left = parseAnd(); + while (peek(TokenType.OR)) { + consume(TokenType.OR); + final VmSearchExpr right = parseAnd(); + left = new VmSearchExpr.Or(left, right); + } + return left; + } + + private VmSearchExpr parseAnd() { + VmSearchExpr left = parsePrimary(); + while (peek(TokenType.AND)) { + consume(TokenType.AND); + final VmSearchExpr right = parsePrimary(); + left = new VmSearchExpr.And(left, right); + } + return left; + } + + private VmSearchExpr parsePrimary() { + if (peek(TokenType.LPAREN)) { + consume(TokenType.LPAREN); + final VmSearchExpr e = parseExpression(); + expect(TokenType.RPAREN); + return e; + } + return parseTerm(); + } + + private VmSearchExpr parseTerm() { + final Token fieldTok = expect(TokenType.IDENT); + final String field = fieldTok.text(); + + if (allowedFields != null && !allowedFields.contains(field)) { + throw new VmSearchParseException("Unsupported search field '" + field + "' at position " + fieldTok.pos()); + } + + expect(TokenType.EQ); + + final Token v = next(); + final String value; + if (v.type() == TokenType.IDENT || v.type() == TokenType.STRING) { + value = v.text(); + } else { + throw new VmSearchParseException("Expected value after '=' at position " + v.pos()); + } + + if (value == null || value.isEmpty()) { + throw new VmSearchParseException("Empty value for field '" + field + "' at position " + v.pos()); + } + + return new VmSearchExpr.Term(field, value); + } + + boolean peek(final TokenType t) { + return tokens.get(k).type() == t; + } + + Token next() { + return tokens.get(k++); + } + + Token expect(final TokenType t) { + final Token tok = next(); + if (tok.type() != t) { + throw new VmSearchParseException("Expected " + t + " at position " + tok.pos() + " but found " + tok.type()); + } + return tok; + } + + Token consume(final TokenType t) { + return expect(t); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java new file mode 100644 index 000000000000..fa67367773eb --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.response; + +import org.apache.cloudstack.veeam.api.dto.Fault; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "fault") +public final class FaultResponse { + public Fault fault; + + public FaultResponse() {} + + public FaultResponse(final Fault fault) { + this.fault = fault; + } + + public static FaultResponse of(final String reason, final String detail) { + return new FaultResponse(new Fault(reason, detail)); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java new file mode 100644 index 000000000000..fc858f51ca05 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.response; + +import java.util.List; + +import org.apache.cloudstack.veeam.api.dto.Vm; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +/** + * Required list response: + * { "vm": [ {..}, {..} ] } + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "vm" }) +@JacksonXmlRootElement(localName = "vms") +public final class VmCollectionResponse { + @JsonProperty("vm") + @JacksonXmlElementWrapper(useWrapping = false) + public List vm; + + public VmCollectionResponse() {} + + public VmCollectionResponse(final List vm) { + this.vm = vm; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java new file mode 100644 index 000000000000..92547b337d5d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.response; + +import org.apache.cloudstack.veeam.api.dto.Vm; + +/** + * Required entity response: + * { "vm": { .. } } + */ +public final class VmEntityResponse { + public Vm vm; + + public VmEntityResponse() {} + + public VmEntityResponse(final Vm vm) { + this.vm = vm; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BasicAuthFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BasicAuthFilter.java new file mode 100644 index 000000000000..22f76b8058e1 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BasicAuthFilter.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.filter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.VeeamControlServlet; + +public class BasicAuthFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + // no-op + } + + @Override + public void destroy() { + // no-op + } + + @Override + public void doFilter( + ServletRequest request, + ServletResponse response, + FilterChain chain + ) throws IOException, ServletException { + + HttpServletRequest req = (HttpServletRequest) request; + HttpServletResponse resp = (HttpServletResponse) response; + + String expectedUser = VeeamControlService.Username.value(); + String expectedPass = VeeamControlService.Password.value(); + + String auth = req.getHeader("Authorization"); + if (auth == null || !auth.regionMatches(true, 0, "Basic ", 0, 6)) { + unauthorized(resp); + return; + } + + String decoded; + try { + decoded = new String( + Base64.getDecoder().decode(auth.substring(6)), + StandardCharsets.UTF_8 + ); + } catch (IllegalArgumentException e) { + unauthorized(resp); + return; + } + + int idx = decoded.indexOf(':'); + if (idx <= 0) { + unauthorized(resp); + return; + } + + String user = decoded.substring(0, idx); + String pass = decoded.substring(idx + 1); + + if (!constantTimeEquals(user, expectedUser) + || !constantTimeEquals(pass, expectedPass)) { + unauthorized(resp); + return; + } + + chain.doFilter(request, response); + } + + private void unauthorized(HttpServletResponse resp) { + throw VeeamControlServlet.Error.unauthorized("Unauthorized"); + } + + private boolean constantTimeEquals(String a, String b) { + byte[] x = a.getBytes(StandardCharsets.UTF_8); + byte[] y = b.getBytes(StandardCharsets.UTF_8); + if (x.length != y.length) return false; + int r = 0; + for (int i = 0; i < x.length; i++) { + r |= x[i] ^ y[i]; + } + return r == 0; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java new file mode 100644 index 000000000000..3aed77efa20b --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.sso; + +import java.io.IOException; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.utils.Negotiation; + +import com.cloud.utils.component.ManagerBase; + +public class SsoService extends ManagerBase implements RouteHandler { + + @Override + public boolean canHandle(String method, String path) { + return path.startsWith("/sso"); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + if ("/sso/oauth/token".equals(path)) { + handleToken(req, resp, outFormat, io); + return; + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + protected void handleToken(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) + throws IOException { + + // Typically POST; you only listed 200/400/401 -> treat others as 400 + if (!"POST".equals(req.getMethod())) { + throw VeeamControlServlet.Error.badRequest("token endpoint requires POST"); + } + + // Assume x-www-form-urlencoded for OAuth token requests (common) + String grantType = req.getParameter("grant_type"); + if (grantType == null || grantType.isBlank()) { + throw VeeamControlServlet.Error.badRequest("Missing parameter: grant_type"); + } + + // NOTE: 401 is normally handled by BasicAuthFilter; keep hook here if you later move auth here. + // if (!authorized) throw VeeamControlServlet.Error.unauthorized("Unauthorized"); + + io.getWriter().write(resp, 200, Map.of( + "access_token", "dummy-token", + "token_type", "bearer", + "expires_in", 3600 + ), outFormat); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Negotiation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Negotiation.java new file mode 100644 index 000000000000..1c82216f113b --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Negotiation.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + + +import javax.servlet.http.HttpServletRequest; + +public final class Negotiation { + + public enum OutFormat { XML, JSON } + + public static OutFormat responseFormat(HttpServletRequest req) { + String accept = req.getHeader("Accept"); + if (accept == null || accept.isBlank() || accept.contains("*/*")) { + return OutFormat.XML; + } + accept = accept.toLowerCase(); + if (accept.contains("application/json")) return OutFormat.JSON; + if (accept.contains("application/xml") || accept.contains("text/xml")) { + return OutFormat.XML; + } + return OutFormat.XML; + } + + public static String contentType(OutFormat fmt) { + return fmt == OutFormat.JSON + ? "application/json" + : "application/xml"; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java new file mode 100644 index 000000000000..a56dde4c75e3 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; + +public class ResponseMapper { + private final ObjectMapper json; + private final XmlMapper xml; + + public ResponseMapper() { + this.json = new ObjectMapper(); + this.xml = new XmlMapper(); + + configure(json); + configure(xml); + } + + private static void configure(final ObjectMapper mapper) { + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + // If you ever add enums etc: + // mapper.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); + // mapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); + } + + public String toJson(final Object value) throws JsonProcessingException { + return json.writeValueAsString(value); + } + + public String toXml(final Object value) throws JsonProcessingException { + return xml.writeValueAsString(value); + } + + public ObjectMapper jsonMapper() { + return json; + } + + public XmlMapper xmlMapper() { + return xml; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java new file mode 100644 index 000000000000..a40ebc860a3c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.api.dto.Fault; +import org.apache.cloudstack.veeam.api.response.FaultResponse; + +public final class ResponseWriter { + + private final ResponseMapper mapper; + + public ResponseWriter(final ResponseMapper mapper) { + this.mapper = mapper; + } + + public void write(final HttpServletResponse resp, final int status, final Object body, final Negotiation.OutFormat fmt) + throws IOException { + + resp.setStatus(status); + + if (body == null) { + resp.setContentLength(0); + return; + } + + final String payload; + final String contentType; + + try { + if (fmt == Negotiation.OutFormat.XML) { + contentType = "application/xml"; + payload = mapper.toXml(body); + } else { + contentType = "application/json"; + payload = mapper.toJson(body); + } + } catch (Exception e) { + // Last-resort fallback + resp.setStatus(500); + resp.setHeader("Content-Type", "text/plain"); + resp.getWriter().write("Internal Server Error"); + return; + } + + resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); + resp.setHeader("Content-Type", contentType); + resp.getWriter().write(payload); + } + + public void writeFault(final HttpServletResponse resp, final int status, final String reason, final String detail, final Negotiation.OutFormat fmt) + throws IOException { + Fault fault = new Fault(reason, detail); + if (fmt == Negotiation.OutFormat.XML) { + write(resp, status, fault, fmt); + } else { + write(resp, status, new FaultResponse(fault), fmt); + } + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties new file mode 100644 index 000000000000..c444a470fb44 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=veeam-control-service +parent=backup \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml new file mode 100644 index 000000000000..0d1d35730312 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + diff --git a/plugins/pom.xml b/plugins/pom.xml index e7d13871285e..b044beaa2c72 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -96,6 +96,7 @@ integrations/cloudian integrations/prometheus integrations/kubernetes-service + integrations/veeam-control-service metrics From 065ec8558966997b87b22d94fc1a5773f8d785a0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 22 Jan 2026 10:18:07 +0530 Subject: [PATCH 002/129] wip Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/veeam/RouteHandler.java | 9 + .../cloudstack/veeam/VeeamControlServer.java | 127 ++++++++- .../cloudstack/veeam/VeeamControlService.java | 2 +- .../cloudstack/veeam/VeeamControlServlet.java | 28 ++ .../cloudstack/veeam/api/ApiService.java | 109 +++++++- .../veeam/api/DataCentersRouteHandler.java | 184 +++++++++++++ .../cloudstack/veeam/api/VmsRouteHandler.java | 7 +- .../DataCenterVOToDataCenterConverter.java | 80 ++++++ .../StoreVOToStorageDomainConverter.java | 248 +++++++++++++++++ .../converter/UserVmJoinVOToVmConverter.java | 25 +- .../apache/cloudstack/veeam/api/dto/Api.java | 62 +++++ .../cloudstack/veeam/api/dto/DataCenter.java | 62 +++++ .../cloudstack/veeam/api/dto/DataCenters.java | 48 ++++ .../veeam/api/dto/EmptyElement.java | 25 ++ .../veeam/api/dto/EmptyElementSerializer.java | 37 +++ .../cloudstack/veeam/api/dto/ProductInfo.java | 36 +++ .../veeam/api/dto/SpecialObjectRef.java | 38 +++ .../veeam/api/dto/SpecialObjects.java | 33 +++ .../cloudstack/veeam/api/dto/Storage.java | 42 +++ .../veeam/api/dto/StorageDomain.java | 100 +++++++ .../veeam/api/dto/StorageDomains.java | 39 +++ .../cloudstack/veeam/api/dto/Summary.java | 39 +++ .../veeam/api/dto/SummaryCount.java | 38 +++ .../veeam/api/dto/SupportedVersions.java | 37 +++ .../cloudstack/veeam/api/dto/Version.java | 42 +++ .../apache/cloudstack/veeam/api/dto/Vm.java | 4 + .../veeam/filter/BearerOrBasicAuthFilter.java | 249 ++++++++++++++++++ .../cloudstack/veeam/sso/SsoService.java | 149 +++++++++-- .../cloudstack/veeam/utils/PathUtil.java | 62 +++++ .../veeam/utils/ResponseMapper.java | 2 + .../veeam/utils/ResponseWriter.java | 5 + .../spring-veeam-control-service-context.xml | 1 + .../utils/server/ServerPropertiesUtil.java | 11 + 33 files changed, 1945 insertions(+), 35 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index 25c4dfbf8f66..5e0db99d1614 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -31,4 +31,13 @@ public interface RouteHandler extends Adapter { boolean canHandle(String method, String path) throws IOException; void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException; + + default String getSanitizedPath(String path) { + // remove query params if exists + int qIdx = path.indexOf('?'); + if (qIdx != -1) { + return path.substring(0, qIdx); + } + return path; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java index aa03cddd2f72..539e89e8473a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -17,19 +17,36 @@ package org.apache.cloudstack.veeam; -import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.EnumSet; +import java.util.Enumeration; import java.util.List; import javax.servlet.DispatcherType; +import javax.servlet.http.HttpServletRequest; -import org.apache.cloudstack.veeam.filter.BasicAuthFilter; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.filter.BearerOrBasicAuthFilter; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.RequestLog; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.jetbrains.annotations.NotNull; public class VeeamControlServer { private static final Logger LOGGER = LogManager.getLogger(VeeamControlServer.class); @@ -49,6 +66,13 @@ public void startIfEnabled() throws Exception { return; } + final String keystorePath = ServerPropertiesUtil.getKeystoreFile(); + final String keystorePassword = ServerPropertiesUtil.getKeystorePassword(); + final String keyManagerPassword = ServerPropertiesUtil.getKeystorePassword(); + final boolean sslConfigured = StringUtils.isNotEmpty(keystorePath) && + StringUtils.isNotEmpty(keystorePassword) && + StringUtils.isNotEmpty(keyManagerPassword) && + Files.exists(Paths.get(keystorePath)); final String bind = VeeamControlService.BindAddress.value(); final int port = VeeamControlService.Port.value(); String ctxPath = VeeamControlService.ContextPath.value(); @@ -56,28 +80,119 @@ public void startIfEnabled() throws Exception { routeHandlers != null ? routeHandlers.size() : 0); - server = new Server(new InetSocketAddress(bind, port)); + server = new Server(); + + if (sslConfigured) { + final SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setKeyStorePath(keystorePath); + sslContextFactory.setKeyStorePassword(keystorePassword); + sslContextFactory.setKeyManagerPassword(keyManagerPassword); + + final HttpConfiguration https = new HttpConfiguration(); + https.setSecureScheme("https"); + https.setSecurePort(port); + https.addCustomizer(new SecureRequestCustomizer()); + + final ServerConnector httpsConnector = new ServerConnector( + server, + new SslConnectionFactory(sslContextFactory, "http/1.1"), + new HttpConnectionFactory(https) + ); + httpsConnector.setHost(bind); + httpsConnector.setPort(port); + server.addConnector(httpsConnector); + + LOGGER.info("Veeam Control API server HTTPS enabled on {}:{}", bind, port); + } else { + final HttpConfiguration http = new HttpConfiguration(); + final ServerConnector httpConnector = new ServerConnector(server, new HttpConnectionFactory(http)); + httpConnector.setHost(bind); + httpConnector.setPort(port); + server.addConnector(httpConnector); + + LOGGER.warn("Veeam Control API server HTTPS is NOT configured (missing keystore path/passwords). " + + "Starting HTTP on {}:{} instead.", bind, port); + } final ServletContextHandler ctx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); ctx.setContextPath(ctxPath); - // Basic Auth for all routes - ctx.addFilter(BasicAuthFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST)); + // Bearer or Basic Auth for all routes + ctx.addFilter(BearerOrBasicAuthFilter.class, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); // Front controller servlet ctx.addServlet(new ServletHolder(new VeeamControlServlet(routeHandlers)), "/*"); - server.setHandler(ctx); + // Create a RequestLog that logs every request handled by the server (all contexts/paths) + server.setHandler(buildContextHandler(ctx)); + server.start(); LOGGER.info("Started Veeam Control API server on {}:{} with context {}", bind, port, ctxPath); } + @NotNull + private static Handler buildContextHandler(ServletContextHandler ctx) { + // Handler for root ('/') path + final ServletContextHandler root = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + root.setContextPath("/"); + root.addServlet(new ServletHolder(new javax.servlet.http.HttpServlet() { + private static final long serialVersionUID = 1L; + + @Override + protected void doGet(javax.servlet.http.HttpServletRequest req, javax.servlet.http.HttpServletResponse resp) + throws java.io.IOException { + resp.setContentType("text/plain"); + resp.setStatus(javax.servlet.http.HttpServletResponse.SC_OK); + resp.getWriter().println("Veeam Control API"); + } + + @Override + protected void doPost(javax.servlet.http.HttpServletRequest req, javax.servlet.http.HttpServletResponse resp) + throws java.io.IOException { + doGet(req, resp); + } + }), "/*"); + + final RequestLog requestLog = (request, response) -> { + final String uri = request.getRequestURI() + + (request.getQueryString() != null ? "?" + request.getQueryString() : ""); + LOGGER.info("Request - remoteAddr: {}, method: {}, uri: {}, headers: {}, status: {}", + request.getRemoteAddr(), + request.getMethod(), + uri, + dumpRequestHeaders(request), + response.getStatus()); + }; + + final RequestLogHandler requestLogHandler = new RequestLogHandler(); + requestLogHandler.setRequestLog(requestLog); + + // Attach both the configured context and the root handler; keep ctx first so contextPath has priority + final HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] { ctx, root }); + requestLogHandler.setHandler(handlers); + return requestLogHandler; + } + public void stop() throws Exception { if (server != null) { server.stop(); server = null; } } + + private static String dumpRequestHeaders(HttpServletRequest request) { + final StringBuilder sb = new StringBuilder(); + final Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + final String name = headerNames.nextElement(); + final Enumeration values = request.getHeaders(name); + while (values.hasMoreElements()) { + sb.append(name).append("=").append(values.nextElement()).append("; "); + } + } + return sb.toString(); + } } \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java index 87233a3ebd8c..adf02be8dd17 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -30,7 +30,7 @@ public interface VeeamControlService extends PluggableService, Configurable { ConfigKey Port = new ConfigKey<>("Advanced", Integer.class, "integration.veeam.control.port", "8090", "Port for Veeam Integration REST API server", false); ConfigKey ContextPath = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.context.path", - "/integrations/veeam", "Context path for Veeam Integration REST API server", false); + "/ovirt-engine", "Context path for Veeam Integration REST API server", false); ConfigKey Username = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.username", "veeam", "Username for Basic Auth on Veeam Integration REST API server", true); ConfigKey Password = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.password", diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java index bf064e27f025..7c38e4cf249e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -58,6 +58,34 @@ protected void service(HttpServletRequest req, HttpServletResponse resp) throws LOGGER.info("Received {} request for {} with out format: {}", method, path, outFormat); + // Add a log to give all info about the request + try { + StringBuilder details = new StringBuilder(); + details.append("Request details: Method: ").append(method).append(", Path: ").append(path); + details.append(", Query: ").append(req.getQueryString() == null ? "" : req.getQueryString()); + details.append(", Headers: "); + java.util.Enumeration headerNames = req.getHeaderNames(); + while (headerNames != null && headerNames.hasMoreElements()) { + String name = headerNames.nextElement(); + details.append(name).append("=").append(req.getHeader(name)).append("; "); + } +// String body = ""; +// if (!"GET".equalsIgnoreCase(method)) { +// StringBuilder bodySb = new StringBuilder(); +// java.io.BufferedReader reader = req.getReader(); +// if (reader != null) { +// String line; +// while ((line = reader.readLine()) != null) { +// bodySb.append(line).append('\n'); +// } +// } +// body = bodySb.toString().trim(); +// } +// details.append(", Body: ").append(body); + LOGGER.debug(details.toString()); + } catch (Exception e) { + LOGGER.debug("Failed to capture request details", e); + } try { if ("/".equals(path)) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index d2a9cf386f01..bb37c300d846 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -18,12 +18,28 @@ package org.apache.cloudstack.veeam.api; import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.dto.Api; +import org.apache.cloudstack.veeam.api.dto.EmptyElement; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.ProductInfo; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.SpecialObjectRef; +import org.apache.cloudstack.veeam.api.dto.SpecialObjects; +import org.apache.cloudstack.veeam.api.dto.Summary; +import org.apache.cloudstack.veeam.api.dto.SummaryCount; +import org.apache.cloudstack.veeam.api.dto.Version; import org.apache.cloudstack.veeam.utils.Negotiation; import com.cloud.utils.component.ManagerBase; @@ -33,12 +49,101 @@ public class ApiService extends ManagerBase implements RouteHandler { @Override public boolean canHandle(String method, String path) { - return path.startsWith("/api"); + return getSanitizedPath(path).startsWith("/api"); } @Override public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - // ToDo: handle root API requests + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleRootApiRequest(req, resp, outFormat, io); + return; + } io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", null, outFormat); } + + private void handleRootApiRequest(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + io.getWriter().write(resp, 200, + createDummyApi(VeeamControlService.ContextPath.value() + BASE_ROUTE), + outFormat); + } + + private static Api createDummyApi(String basePath) { + Api api = new Api(); + + /* ---------------- Links ---------------- */ + List links = new ArrayList<>(); + add(links, basePath + "/clusters", "clusters"); + add(links, basePath + "/clusters?search={query}", "clusters/search"); + add(links, basePath + "/datacenters", "datacenters"); + add(links, basePath + "/datacenters?search={query}", "datacenters/search"); + add(links, basePath + "/events", "events"); + add(links, basePath + "/events;from={event_id}?search={query}", "events/search"); + add(links, basePath + "/hosts", "hosts"); + add(links, basePath + "/hosts?search={query}", "hosts/search"); + add(links, basePath + "/networks", "networks"); + add(links, basePath + "/networks?search={query}", "networks/search"); + add(links, basePath + "/storagedomains", "storagedomains"); + add(links, basePath + "/storagedomains?search={query}", "storagedomains/search"); + add(links, basePath + "/templates", "templates"); + add(links, basePath + "/templates?search={query}", "templates/search"); + add(links, basePath + "/vms", "vms"); + add(links, basePath + "/vms?search={query}", "vms/search"); + add(links, basePath + "/disks", "disks"); + add(links, basePath + "/disks?search={query}", "disks/search"); + + api.link = links; + + /* ---------------- Engine backup ---------------- */ + api.engineBackup = new EmptyElement(); + + /* ---------------- Product info ---------------- */ + ProductInfo productInfo = new ProductInfo(); + productInfo.instanceId = UUID.randomUUID().toString(); + productInfo.name = "oVirt Engine"; + + Version version = new Version(); + version.build = "8"; + version.fullVersion = "4.5.8-0.master.fake.el9"; + version.major = 4; + version.minor = 5; + version.revision = 0; + + productInfo.version = version; + api.productInfo = productInfo; + + /* ---------------- Special objects ---------------- */ + SpecialObjects specialObjects = new SpecialObjects(); + specialObjects.blankTemplate = new SpecialObjectRef( + basePath + "/templates/00000000-0000-0000-0000-000000000000", + "00000000-0000-0000-0000-000000000000" + ); + specialObjects.rootTag = new SpecialObjectRef( + basePath + "/tags/00000000-0000-0000-0000-000000000000", + "00000000-0000-0000-0000-000000000000" + ); + api.specialObjects = specialObjects; + + /* ---------------- Summary ---------------- */ + Summary summary = new Summary(); + summary.hosts = new SummaryCount(1, 1); + summary.storageDomains = new SummaryCount(1, 2); + summary.users = new SummaryCount(1, 1); + summary.vms = new SummaryCount(1, 8); + api.summary = summary; + + /* ---------------- Time ---------------- */ + api.time = OffsetDateTime.now(ZoneOffset.ofHours(2)).toInstant().toEpochMilli(); + + /* ---------------- Users ---------------- */ + String userId = UUID.randomUUID().toString(); + api.authenticatedUser = Ref.of(basePath + "/users/" + userId, userId); + api.effectiveUser = Ref.of(basePath + "/users/" + userId, userId); + + return api; + } + + private static void add(List links, String href, String rel) { + links.add(new Link(href, rel)); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java new file mode 100644 index 000000000000..d297fe9b516d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.DataCenterVOToDataCenterConverter; +import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; +import org.apache.cloudstack.veeam.api.dto.DataCenter; +import org.apache.cloudstack.veeam.api.dto.DataCenters; +import org.apache.cloudstack.veeam.api.dto.StorageDomain; +import org.apache.cloudstack.veeam.api.dto.StorageDomains; +import org.apache.cloudstack.veeam.api.request.VmListQuery; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.api.query.dao.ImageStoreJoinDao; +import com.cloud.api.query.dao.StoragePoolJoinDao; +import com.cloud.api.query.vo.ImageStoreJoinVO; +import com.cloud.api.query.vo.StoragePoolJoinVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class DataCentersRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/datacenters"; + private static final int DEFAULT_MAX = 50; + private static final int HARD_CAP_MAX = 1000; + private static final int DEFAULT_PAGE = 1; + + @Inject + DataCenterDao dataCenterDao; + + @Inject + StoragePoolJoinDao storagePoolJoinDao; + + @Inject + ImageStoreJoinDao imageStoreJoinDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/datacenters/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + if ("storagedomains".equals(idAndSubPath.second())) { + handleGetStorageDomainsByDcId(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + /** + * Matches /api/datacenters/{id} where {id} is a single path segment (no extra '/'). + * Returns id or null. + */ + private static String matchSinglePathParam(final String path, final String prefix) { + if (!path.startsWith(prefix)) return null; + final String rest = path.substring(prefix.length()); // after "/api/datacenters/" + if (rest.isEmpty()) return null; + if (rest.contains("/")) return null; // ensure only 1 segment + return rest; + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = DataCenterVOToDataCenterConverter.toDCList(listDCs()); + final DataCenters response = new DataCenters(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + private static VmListQuery fromRequest(final HttpServletRequest req) { + final VmListQuery q = new VmListQuery(); + q.setSearch(req.getParameter("search")); + q.setMax(parseIntOrNull(req.getParameter("max"))); + q.setPage(parseIntOrNull(req.getParameter("page"))); + return q; + } + + private static Integer parseIntOrNull(final String s) { + if (s == null || s.trim().isEmpty()) return null; + try { + return Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + return Integer.valueOf(-1); // will be rejected by validation above + } + } + + protected List listDCs() { + return dataCenterDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(id); + if (dataCenterVO == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + DataCenter response = DataCenterVOToDataCenterConverter.toDataCenter(dataCenterVO); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listStoragePoolsByDcId(final long dcId) { + return storagePoolJoinDao.listAll(); + } + + protected List listImageStoresByDcId(final long dcId) { + return imageStoreJoinDao.listAll(); + } + + public void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(id); + if (dataCenterVO == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + List storageDomains = StoreVOToStorageDomainConverter.toStorageDomainListFromPools(listStoragePoolsByDcId(dataCenterVO.getId())); + storageDomains.addAll(StoreVOToStorageDomainConverter.toStorageDomainListFromStores(listImageStoresByDcId(dataCenterVO.getId()))); + + StorageDomains response = new StorageDomains(storageDomains); + + io.getWriter().write(resp, 200, response, outFormat); + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 166794e37bb0..23f626e326eb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -68,13 +68,14 @@ public int priority() { @Override public boolean canHandle(String method, String path) { - return path.startsWith(BASE_ROUTE); + return getSanitizedPath(path).startsWith(BASE_ROUTE); } @Override public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final String method = req.getMethod(); - if (path.equals(BASE_ROUTE)) { + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { if (!"GET".equalsIgnoreCase(method)) { io.methodNotAllowed(resp, "GET", outFormat); return; @@ -84,7 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } // /api/vms/{id} - final String vmId = matchSinglePathParam(path, BASE_ROUTE + "/"); + final String vmId = matchSinglePathParam(sanitizedPath, BASE_ROUTE + "/"); if (vmId != null) { if (!"GET".equalsIgnoreCase(method)) { io.methodNotAllowed(resp, "GET", outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java new file mode 100644 index 000000000000..395bb233ea53 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; +import org.apache.cloudstack.veeam.api.dto.DataCenter; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.SupportedVersions; +import org.apache.cloudstack.veeam.api.dto.Version; + +import com.cloud.dc.DataCenterVO; +import com.cloud.org.Grouping; + +public class DataCenterVOToDataCenterConverter { + public static DataCenter toDataCenter(final DataCenterVO zone) { + final String id = zone.getUuid(); + final String basePath = VeeamControlService.ContextPath.value(); + final String href = basePath + DataCentersRouteHandler.BASE_ROUTE + "/datacenters/" + id; + + final DataCenter dc = new DataCenter(); + + // ---- Identity ---- + dc.id = id; + dc.href = href; + dc.name = zone.getName(); + dc.description = zone.getDescription(); + + // ---- State ---- + dc.status = Grouping.AllocationState.Enabled.equals(zone.getAllocationState()) ? "up" : "down"; + dc.local = "false"; + dc.quotaMode = "disabled"; + dc.storageFormat = "v5"; + + // ---- Versions (static but valid) ---- + final Version v48 = new Version(); + v48.major = 4; + v48.minor = 8; + dc.version = v48; + dc.supportedVersions = new SupportedVersions(List.of(v48)); + + // ---- mac_pool (static placeholder) ---- + dc.macPool = Ref.of(basePath + "/macpools/default","default"); + + // ---- Related links ---- + dc.link = Arrays.asList( + new Link(href + "/clusters", "clusters"), + new Link(href + "/networks", "networks"), + new Link(href + "/storagedomains", "storagedomains") + ); + + return dc; + } + + public static List toDCList(final List srcList) { + return srcList.stream() + .map(DataCenterVOToDataCenterConverter::toDataCenter) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java new file mode 100644 index 000000000000..071ebc92c142 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; +import org.apache.cloudstack.veeam.api.dto.DataCenter; +import org.apache.cloudstack.veeam.api.dto.DataCenters; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Storage; +import org.apache.cloudstack.veeam.api.dto.StorageDomain; + +import com.cloud.api.query.vo.ImageStoreJoinVO; +import com.cloud.api.query.vo.StoragePoolJoinVO; + +public class StoreVOToStorageDomainConverter { + + /** Primary storage -> oVirt storage_domain (type=data) */ + public static StorageDomain toStorageDomain(final StoragePoolJoinVO pool) { + final String basePath = VeeamControlService.ContextPath.value(); + + final String id = pool.getUuid(); + + StorageDomain sd = new StorageDomain(); + sd.id = id; + final String href = href(basePath, ApiService.BASE_ROUTE + "/storagedomains/" + id); + sd.href = href; + + sd.name = pool.getName(); + sd.description = ""; // oVirt often returns empty string + sd.comment = ""; + + // oVirt sample returns numbers as strings + sd.available = Long.toString(pool.getCapacityBytes() - pool.getUsedBytes()); + sd.used = Long.toString(pool.getUsedBytes()); + sd.committed = Long.toString(pool.getCapacityBytes()); + + sd.type = "data"; + sd.status = mapPoolStatus(pool); // "active"/"inactive"/"maintenance" (approx) + sd.master = "true"; // if you don’t have a concept, choose stable default + sd.backup = "false"; + + sd.blockSize = "512"; // stable default unless you can compute it + sd.externalStatus = "ok"; + sd.storageFormat = "v5"; + + sd.discardAfterDelete = "false"; + sd.wipeAfterDelete = "false"; + sd.supportsDiscard = "false"; + sd.supportsDiscardZeroesData = "false"; + + sd.warningLowSpaceIndicator = "10"; + sd.criticalSpaceActionBlocker = "5"; + + // Nested storage (try to extract if available) + sd.storage = buildPrimaryStorage(pool); + + // dc attachment + String dcId = pool.getZoneUuid(); + DataCenter dc = new DataCenter(); + dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + dcId); + dc.id = dcId; + sd.dataCenters = new DataCenters(List.of(dc)); + + sd.link = defaultStorageDomainLinks(href, true, /*includeTemplates*/ true); + + return sd; + } + + public static List toStorageDomainListFromPools(final List pools) { + return pools.stream().map(StoreVOToStorageDomainConverter::toStorageDomain).collect(Collectors.toList()); + } + + /** Secondary/Image store -> oVirt storage_domain (type=image) */ + public static StorageDomain toStorageDomain(final ImageStoreJoinVO store) { + final String basePath = VeeamControlService.ContextPath.value(); + + final String id = store.getUuid(); + + StorageDomain sd = new StorageDomain(); + sd.id = id; + final String href = href(basePath, ApiService.BASE_ROUTE + "/storagedomains/" + id); + sd.href = href; + + sd.name = store.getName(); + sd.description = ""; + sd.comment = ""; + + // Many image repos don’t have these values readily; keep as "0" or omit (null) + sd.committed = "0"; + sd.available = null; // oVirt’s glance example omitted available/used + sd.used = null; + + sd.type = "image"; + sd.status = "unattached"; // matches your sample for glance-like repo + sd.master = "false"; + sd.backup = "false"; + + sd.blockSize = "512"; + sd.externalStatus = "ok"; + sd.storageFormat = "v1"; + + sd.discardAfterDelete = "false"; + sd.wipeAfterDelete = "false"; + sd.supportsDiscard = "false"; + sd.supportsDiscardZeroesData = "false"; + + sd.warningLowSpaceIndicator = "0"; + sd.criticalSpaceActionBlocker = "0"; + + sd.storage = buildImageStoreStorage(store); + + // Optionally include dc attachment (your first object had it; second didn’t) + String dcId = store.getZoneUuid(); + DataCenter dc = new DataCenter(); + dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + dcId); + dc.id = dcId; + sd.dataCenters = new DataCenters(List.of(dc)); + + sd.link = defaultStorageDomainLinks(href, false, /*includeTemplates*/ false); + + return sd; + } + + public static List toStorageDomainListFromStores(final List stores) { + return stores.stream().map(StoreVOToStorageDomainConverter::toStorageDomain).collect(Collectors.toList()); + } + + // ----------- Helpers ----------- + + private static Storage buildPrimaryStorage(StoragePoolJoinVO pool) { + Storage st = new Storage(); + st.type = mapPrimaryStorageType(pool); + + // If you can parse details/url, fill these. If not, keep empty strings like oVirt. + // For NFS pools in CloudStack, URL is often like: nfs://10.0.32.4/path or 10.0.32.4:/path + String url = null; + try { + url = pool.getHostAddress(); // sometimes exists in VO; if not, ignore + } catch (Exception ignored) { } + + if ("nfs".equals(st.type)) { + // best-effort placeholders + st.address = ""; // fill if you can parse + st.path = ""; // fill if you can parse + st.mountOptions = ""; + st.nfsVersion = "auto"; + } + return st; + } + + private static Storage buildImageStoreStorage(ImageStoreJoinVO store) { + Storage st = new Storage(); + + // Match your sample: glance store => type=glance + // If you want "nfs" for secondary, map based on provider/protocol instead. + st.type = mapImageStorageType(store); + + if ("nfs".equals(st.type)) { + st.address = ""; + st.path = ""; + st.mountOptions = ""; + st.nfsVersion = "auto"; + } + return st; + } + + private static List defaultStorageDomainLinks(String basePath, boolean includeDisks, boolean includeTemplates) { + // Mirrors the rels you pasted; keep stable order. + // You can add/remove based on what endpoints you actually implement. + List common = new java.util.ArrayList<>(); + common.add(new Link("diskprofiles", href(basePath, "/diskprofiles"))); + if (includeDisks) { + common.add(new Link("disks", href(basePath, "/disks"))); + common.add(new Link("storageconnections", href(basePath, "/storageconnections"))); + } + common.add(new Link("permissions", href(basePath, "/permissions"))); + if (includeTemplates) { + common.add(new Link("templates", href(basePath, "/templates"))); + common.add(new Link("vms", href(basePath, "/vms"))); + } else { + common.add(new Link("images", href(basePath, "/images"))); + } + common.add(new Link("disksnapshots", href(basePath, "/disksnapshots"))); + return common; + } + + private static String mapPoolStatus(StoragePoolJoinVO pool) { + // This is approximate; adjust if you have better signals. + try { + Object status = pool.getStatus(); // often StoragePoolStatus enum + if (status != null) { + String s = status.toString().toLowerCase(); + if (s.contains("up") || s.contains("enabled")) return "active"; + if (s.contains("maintenance")) return "maintenance"; + } + } catch (Exception ignored) { } + return "inactive"; + } + + private static String mapPrimaryStorageType(StoragePoolJoinVO pool) { + try { + Object t = pool.getPoolType(); // often StoragePoolType enum + if (t != null) { + String s = t.toString().toLowerCase(); + if (s.contains("networkfilesystem") || s.contains("nfs")) return "nfs"; + if (s.contains("iscsi")) return "iscsi"; + if (s.contains("filesystem")) return "posixfs"; + if (s.contains("rbd") || s.contains("ceph")) return "cinder"; // not perfect; pick stable + } + } catch (Exception ignored) { } + return "unknown"; + } + + private static String mapImageStorageType(ImageStoreJoinVO store) { + // If your secondary store is S3/NFS/etc, you may want different mapping. + // For your oVirt sample, "glance" is used for an image repo. + try { + String provider = store.getProviderName(); // may exist + if (provider != null && provider.toLowerCase().contains("glance")) return "glance"; + } catch (Exception ignored) { } + return "glance"; + } + + private static String href(String baseUrl, String path) { + if (baseUrl.endsWith("/")) baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + return baseUrl + path; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index ba5c831169d0..760ab5c758c4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -25,7 +25,10 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Bios; import org.apache.cloudstack.veeam.api.dto.Cpu; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Os; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -76,13 +79,25 @@ public static Vm toVm(final UserVmJoinVO src) { basePath + ApiService.BASE_ROUTE, "cluster", src.getHostUuid()); - dst.memory = (long) src.getRamSize(); + dst.memory = src.getRamSize() * 1024L * 1024L; dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); - dst.os = null; - dst.bios = null; - dst.actions = null; - dst.link = null; + dst.os = new Os(); + dst.os.type = src.getGuestOsId() % 2 == 0 + ? "windows" + : "linux"; + dst.bios = new Bios(); + dst.bios.type = "legacy"; + dst.type = "server"; + dst.origin = "ovirt"; + dst.actions = null;dst.link = List.of( + new Link("diskattachments", + dst.href + "/diskattachments"), + new Link("nics", + dst.href + "/nics"), + new Link("snapshots", + dst.href + "/snapshots") + ); return dst; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java new file mode 100644 index 000000000000..2571f32111f4 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +/** + * Root response for GET /ovirt-engine/api + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "api") +public final class Api { + + // repeated + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + // (empty element) + @JacksonXmlProperty(localName = "engine_backup") + public EmptyElement engineBackup; + + @JacksonXmlProperty(localName = "product_info") + public ProductInfo productInfo; + + @JacksonXmlProperty(localName = "special_objects") + public SpecialObjects specialObjects; + + @JacksonXmlProperty(localName = "summary") + public Summary summary; + + // Keep as String to avoid timezone/date parsing friction; you control formatting. + @JacksonXmlProperty(localName = "time") + public Long time; + + @JacksonXmlProperty(localName = "authenticated_user") + public Ref authenticatedUser; + + @JacksonXmlProperty(localName = "effective_user") + public Ref effectiveUser; + + public Api() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java new file mode 100644 index 000000000000..acba378032cb --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "data_center") +public final class DataCenter { + + // keep strings to match oVirt JSON ("false", "disabled", "up", "v5", etc.) + public String local; + + @JsonProperty("quota_mode") + public String quotaMode; + + public String status; + + @JsonProperty("storage_format") + public String storageFormat; + + @JsonProperty("supported_versions") + public SupportedVersions supportedVersions; + + public Version version; + + @JsonProperty("mac_pool") + public Ref macPool; + + public Actions actions; + + public String name; + public String description; + + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + public String href; + public String id; + + public DataCenter() {} +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java new file mode 100644 index 000000000000..24e6f2884256 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +/** + * Root collection wrapper: + * { + * "data_center": [ { ... } ] + * } + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "data_centers") +@JsonPropertyOrder({ "data_center" }) +public final class DataCenters { + + @JsonProperty("data_center") + @JacksonXmlElementWrapper(useWrapping = false) + public List dataCenter; + + public DataCenters() {} + public DataCenters(final List dataCenter) { + this.dataCenter = dataCenter; + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java new file mode 100644 index 000000000000..54d65d8529bc --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = EmptyElementSerializer.class) +public final class EmptyElement { + public EmptyElement() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java new file mode 100644 index 000000000000..4b6a407aecf3 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +/** + * Serializes as an "empty object". + * With Jackson XML this becomes an empty element: . + */ +public final class EmptyElementSerializer extends JsonSerializer { + @Override + public void serialize(EmptyElement value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeStartObject(); + gen.writeEndObject(); + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java new file mode 100644 index 000000000000..e3618b0e6f9c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class ProductInfo { + + @JacksonXmlProperty(localName = "instance_id") + public String instanceId; + + @JacksonXmlProperty(localName = "name") + public String name; + + @JacksonXmlProperty(localName = "version") + public Version version; + + public ProductInfo() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java new file mode 100644 index 000000000000..39b52c8bd0d2 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class SpecialObjectRef { + + @JacksonXmlProperty(isAttribute = true, localName = "href") + public String href; + + @JacksonXmlProperty(isAttribute = true, localName = "id") + public String id; + + public SpecialObjectRef() {} + + public SpecialObjectRef(String href, String id) { + this.href = href; + this.id = id; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java new file mode 100644 index 000000000000..dc747fa177e1 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class SpecialObjects { + + @JacksonXmlProperty(localName = "blank_template") + public SpecialObjectRef blankTemplate; + + @JacksonXmlProperty(localName = "root_tag") + public SpecialObjectRef rootTag; + + public SpecialObjects() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java new file mode 100644 index 000000000000..edf411ec9be1 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Storage { + + public String type; // nfs / glance + + // nfs-ish fields (optional) + public String address; + public String path; + + @JsonProperty("mount_options") + @JacksonXmlProperty(localName = "mount_options") + public String mountOptions; + + @JsonProperty("nfs_version") + @JacksonXmlProperty(localName = "nfs_version") + public String nfsVersion; + + public Storage() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java new file mode 100644 index 000000000000..0b4663fd0395 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "storage_domain") +public final class StorageDomain { + + // Identifiers + public String id; + public String href; + + public String name; + public String description; + public String comment; + + // oVirt returns these as strings in your sample + public String available; + public String used; + public String committed; + + @JsonProperty("block_size") + @JacksonXmlProperty(localName = "block_size") + public String blockSize; + + @JsonProperty("warning_low_space_indicator") + @JacksonXmlProperty(localName = "warning_low_space_indicator") + public String warningLowSpaceIndicator; + + @JsonProperty("critical_space_action_blocker") + @JacksonXmlProperty(localName = "critical_space_action_blocker") + public String criticalSpaceActionBlocker; + + public String status; // e.g. "unattached" (optional in your first object) + public String type; // data / image / iso / export + + public String master; // "true"/"false" + public String backup; // "true"/"false" + + @JsonProperty("external_status") + @JacksonXmlProperty(localName = "external_status") + public String externalStatus; // "ok" + + @JsonProperty("storage_format") + @JacksonXmlProperty(localName = "storage_format") + public String storageFormat; // v5 / v1 + + @JsonProperty("discard_after_delete") + @JacksonXmlProperty(localName = "discard_after_delete") + public String discardAfterDelete; + + @JsonProperty("wipe_after_delete") + @JacksonXmlProperty(localName = "wipe_after_delete") + public String wipeAfterDelete; + + @JsonProperty("supports_discard") + @JacksonXmlProperty(localName = "supports_discard") + public String supportsDiscard; + + @JsonProperty("supports_discard_zeroes_data") + @JacksonXmlProperty(localName = "supports_discard_zeroes_data") + public String supportsDiscardZeroesData; + + // Nested + public Storage storage; + + @JsonProperty("data_centers") + @JacksonXmlProperty(localName = "data_centers") + public DataCenters dataCenters; + + public Actions actions; + + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + public StorageDomain() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java new file mode 100644 index 000000000000..7fffa8f9a8f2 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "storage_domains") +public final class StorageDomains { + + @JsonProperty("storage_domain") + @JacksonXmlElementWrapper(useWrapping = false) + public List storageDomain; + + public StorageDomains() {} + public StorageDomains(List storageDomain) { + this.storageDomain = storageDomain; + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java new file mode 100644 index 000000000000..992590f5f978 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Summary { + + @JacksonXmlProperty(localName = "hosts") + public SummaryCount hosts; + + @JacksonXmlProperty(localName = "storage_domains") + public SummaryCount storageDomains; + + @JacksonXmlProperty(localName = "users") + public SummaryCount users; + + @JacksonXmlProperty(localName = "vms") + public SummaryCount vms; + + public Summary() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java new file mode 100644 index 000000000000..a0266a2b89a7 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class SummaryCount { + + @JacksonXmlProperty(localName = "active") + public Integer active; + + @JacksonXmlProperty(localName = "total") + public Integer total; + + public SummaryCount() {} + + public SummaryCount(Integer active, Integer total) { + this.active = active; + this.total = total; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java new file mode 100644 index 000000000000..7c73b9e5d949 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class SupportedVersions { + + @JsonProperty("version") + @JacksonXmlElementWrapper(useWrapping = false) + public List version; + + public SupportedVersions() {} + public SupportedVersions(final List version) { + this.version = version; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java new file mode 100644 index 000000000000..cd4601838d1a --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class Version { + + @JacksonXmlProperty(localName = "build") + public String build; + + @JacksonXmlProperty(localName = "full_version") + public String fullVersion; + + @JacksonXmlProperty(localName = "major") + public Integer major; + + @JacksonXmlProperty(localName = "minor") + public Integer minor; + + @JacksonXmlProperty(localName = "revision") + public Integer revision; + + public Version() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index ce1b64e53039..4bba580a9712 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -61,6 +61,10 @@ public final class Vm { public Os os; public Bios bios; + public boolean stateless; // true|false + public String type; // "server" + public String origin; // "ovirt" + public Actions actions; // actions.link[] @JacksonXmlElementWrapper(useWrapping = false) public List link; // related resources diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java new file mode 100644 index 000000000000..5a83299207de --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java @@ -0,0 +1,249 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.filter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Base64; +import java.util.List; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.VeeamControlService; + +public class BearerOrBasicAuthFilter implements Filter { + + // Keep these aligned with SsoService (move to ConfigKeys later) + public static final List REQUIRED_SCOPES = List.of("ovirt-app-admin", "ovirt-app-portal"); + public static final String ISSUER = "veeam-control"; + private static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; + + @Override public void init(FilterConfig filterConfig) {} + @Override public void destroy() {} + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + + final HttpServletRequest req = (HttpServletRequest) request; + final HttpServletResponse resp = (HttpServletResponse) response; + + final String auth = req.getHeader("Authorization"); + if (auth != null && auth.regionMatches(true, 0, "Bearer ", 0, 7)) { + final String token = auth.substring(7).trim(); + if (token.isEmpty()) { + unauthorized(req, resp, "invalid_token", "Missing Bearer token"); + return; + } + if (!verifyJwtHs256(token)) { + unauthorized(req, resp, "invalid_token", "Invalid or expired token"); + return; + } + chain.doFilter(request, response); + return; + } + + // Optional fallback: Basic (handy for manual testing). + if (auth != null && auth.regionMatches(true, 0, "Basic ", 0, 6)) { + if (!verifyBasic(auth.substring(6))) { + unauthorized(req, resp, "invalid_client", "Invalid Basic credentials"); + return; + } + chain.doFilter(request, response); + return; + } + + unauthorized(req, resp, "invalid_token", "Missing Authorization"); + } + + private boolean verifyBasic(String b64) { + final String expectedUser = VeeamControlService.Username.value(); + final String expectedPass = VeeamControlService.Password.value(); + + final String decoded; + try { + decoded = new String(Base64.getDecoder().decode(b64), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + return false; + } + + final int idx = decoded.indexOf(':'); + if (idx <= 0) return false; + + final String user = decoded.substring(0, idx); + final String pass = decoded.substring(idx + 1); + + return constantTimeEquals(user, expectedUser) && constantTimeEquals(pass, expectedPass); + } + + /** + * Minimal JWT verification: + * - HS256 signature + * - "iss" matches + * - "exp" not expired + * - "scope" contains REQUIRED_SCOPES (space-separated) + * + * NOTE: This does not parse JSON robustly; it’s sufficient for the token you mint in SsoService. + * If you want robust parsing, switch to Nimbus and keep the rest the same. + */ + private boolean verifyJwtHs256(String token) { + final String[] parts = token.split("\\."); + if (parts.length != 3) return false; + + final String headerB64 = parts[0]; + final String payloadB64 = parts[1]; + final String sigB64 = parts[2]; + + final byte[] expectedSig; + try { + expectedSig = hmacSha256((headerB64 + "." + payloadB64).getBytes(StandardCharsets.UTF_8), + HMAC_SECRET.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + return false; + } + + final byte[] providedSig; + try { + providedSig = Base64.getUrlDecoder().decode(sigB64); + } catch (IllegalArgumentException e) { + return false; + } + + if (!constantTimeEquals(expectedSig, providedSig)) return false; + + final String payloadJson; + try { + payloadJson = new String(Base64.getUrlDecoder().decode(payloadB64), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + return false; + } + + // Super small “claims” extraction (good enough for our minting format) + final String iss = JsonMini.getString(payloadJson, "iss"); + final String scope = JsonMini.getString(payloadJson, "scope"); + final Long exp = JsonMini.getLong(payloadJson, "exp"); + + if (iss == null || !ISSUER.equals(iss)) return false; + if (exp == null || Instant.now().getEpochSecond() >= exp) return false; + if (scope == null || !hasRequiredScopes(scope)) return false; + + return true; + } + + private static boolean hasRequiredScopes(String scope) { + String[] scopes = scope.split("\\s+"); + for (String required : REQUIRED_SCOPES) { + if (!hasScope(scopes, required)) return false; + } + return true; + } + + private static boolean hasScope(String[] scopes, String required) { + for (String scope : scopes) { + if (scope.equals(required)) { + return true; + } + } + return false; + } + + private static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { + final Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data); + } + + private static void unauthorized(HttpServletRequest req, HttpServletResponse resp, + String error, String desc) throws IOException { + + // IMPORTANT: don’t throw (your current filter throws and Jetty turns it into 500) :contentReference[oaicite:3]{index=3} + resp.resetBuffer(); + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + + // Helpful for OAuth clients: + resp.setHeader("WWW-Authenticate", + "Bearer realm=\"Veeam Integration\", error=\"" + esc(error) + "\", error_description=\"" + esc(desc) + "\""); + + final String accept = req.getHeader("Accept"); + final boolean wantsJson = accept != null && accept.toLowerCase().contains("application/json"); + + resp.setCharacterEncoding("UTF-8"); + if (wantsJson) { + resp.setContentType("application/json; charset=UTF-8"); + resp.getWriter().write("{\"error\":\"" + esc(error) + "\",\"error_description\":\"" + esc(desc) + "\"}"); + } else { + resp.setContentType("text/html; charset=UTF-8"); + resp.getWriter().write("ErrorUnauthorized"); + } + resp.getWriter().flush(); + } + + private static String esc(String s) { + return s == null ? "" : s.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private static boolean constantTimeEquals(String a, String b) { + if (a == null || b == null) return false; + return constantTimeEquals(a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8)); + } + + private static boolean constantTimeEquals(byte[] x, byte[] y) { + if (x.length != y.length) return false; + int r = 0; + for (int i = 0; i < x.length; i++) r |= x[i] ^ y[i]; + return r == 0; + } + + // Tiny JSON extractor for flat string/number claims. Good enough for tokens you mint. + static final class JsonMini { + static String getString(String json, String key) { + final String needle = "\"" + key + "\":"; + int i = json.indexOf(needle); + if (i < 0) return null; + i += needle.length(); + while (i < json.length() && Character.isWhitespace(json.charAt(i))) i++; + if (i >= json.length() || json.charAt(i) != '"') return null; + i++; + int j = json.indexOf('"', i); + if (j < 0) return null; + return json.substring(i, j); + } + + static Long getLong(String json, String key) { + final String needle = "\"" + key + "\":"; + int i = json.indexOf(needle); + if (i < 0) return null; + i += needle.length(); + while (i < json.length() && Character.isWhitespace(json.charAt(i))) i++; + int j = i; + while (j < json.length() && (Character.isDigit(json.charAt(j)))) j++; + if (j == i) return null; + return Long.parseLong(json.substring(i, j)); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java index 3aed77efa20b..fcd984ffce08 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -18,27 +18,40 @@ package org.apache.cloudstack.veeam.sso; import java.io.IOException; +import java.time.Instant; +import java.util.HashMap; import java.util.Map; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.filter.BearerOrBasicAuthFilter; import org.apache.cloudstack.veeam.utils.Negotiation; import com.cloud.utils.component.ManagerBase; public class SsoService extends ManagerBase implements RouteHandler { + private static final String BASE_ROUTE = "/sso"; + private static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; // >= 32 chars recommended + private static final long DEFAULT_TTL_SECONDS = 3600; + + // Replace with your real credential validation (CloudStack account, config, etc.) + private final PasswordAuthenticator authenticator = new StaticPasswordAuthenticator(); @Override public boolean canHandle(String method, String path) { - return path.startsWith("/sso"); + return getSanitizedPath(path).startsWith(BASE_ROUTE); } @Override public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - if ("/sso/oauth/token".equals(path)) { + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE + "/oauth/token")) { handleToken(req, resp, outFormat, io); return; } @@ -46,27 +59,127 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - protected void handleToken(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) - throws IOException { + protected void handleToken(HttpServletRequest req, HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - // Typically POST; you only listed 200/400/401 -> treat others as 400 - if (!"POST".equals(req.getMethod())) { - throw VeeamControlServlet.Error.badRequest("token endpoint requires POST"); + if (!"POST".equalsIgnoreCase(req.getMethod())) { + // oVirt-like: 405 is fine; if you strictly want 400, change to SC_BAD_REQUEST + resp.setHeader("Allow", "POST"); + io.getWriter().write(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, + Map.of("error", "method_not_allowed", "message", "token endpoint requires POST"), outFormat); + return; } - // Assume x-www-form-urlencoded for OAuth token requests (common) - String grantType = req.getParameter("grant_type"); - if (grantType == null || grantType.isBlank()) { - throw VeeamControlServlet.Error.badRequest("Missing parameter: grant_type"); + // OAuth password grant uses x-www-form-urlencoded. With servlet containers this usually populates getParameter(). + final String grantType = trimToNull(req.getParameter("grant_type")); + final String scope = trimToNull(req.getParameter("scope")); // typically "ovirt-app-api" + final String username = trimToNull(req.getParameter("username")); + final String password = trimToNull(req.getParameter("password")); + + if (grantType == null) { + io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, + Map.of("error", "invalid_request", "error_description", "Missing parameter: grant_type"), outFormat); + return; + } + if (!"password".equals(grantType)) { + io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, + Map.of("error", "unsupported_grant_type", "error_description", "Only grant_type=password is supported"), outFormat); + return; + } + if (username == null || password == null) { + io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, + Map.of("error", "invalid_request", "error_description", "Missing username/password"), outFormat); + return; } - // NOTE: 401 is normally handled by BasicAuthFilter; keep hook here if you later move auth here. - // if (!authorized) throw VeeamControlServlet.Error.unauthorized("Unauthorized"); + if (!authenticator.authenticate(username, password)) { + // 401 for bad creds + io.getWriter().write(resp, HttpServletResponse.SC_UNAUTHORIZED, + Map.of("error", "invalid_grant", "error_description", "Invalid credentials"), outFormat); + return; + } + + final String effectiveScope = (scope == null) ? "ovirt-app-api" : scope; + + final long ttl = DEFAULT_TTL_SECONDS; + final String token; + try { + token = JwtUtil.issueHs256Jwt(BearerOrBasicAuthFilter.ISSUER, username, effectiveScope, ttl, HMAC_SECRET); + } catch (Exception e) { + io.getWriter().write(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + Map.of("error", "server_error", "error_description", "Failed to issue token"), outFormat); + return; + } + + final Map payload = new HashMap<>(); + payload.put("access_token", token); + payload.put("token_type", "bearer"); + payload.put("expires_in", ttl); + payload.put("scope", effectiveScope); + + io.getWriter().write(resp, HttpServletResponse.SC_OK, payload, outFormat); + } + + private static String trimToNull(String s) { + if (s == null) return null; + s = s.trim(); + return s.isEmpty() ? null : s; + } - io.getWriter().write(resp, 200, Map.of( - "access_token", "dummy-token", - "token_type", "bearer", - "expires_in", 3600 - ), outFormat); + // ---------- Minimal auth helpers (replace later) ---------- + + interface PasswordAuthenticator { + boolean authenticate(String username, String password); + } + + static final class StaticPasswordAuthenticator implements PasswordAuthenticator { + StaticPasswordAuthenticator() { + } + @Override + public boolean authenticate(String username, String password) { + return VeeamControlService.Username.value().equals(username) && + VeeamControlService.Password.value().equals(password); + } + } + + // ---------- Minimal JWT HS256 without extra libs ---------- + // (If you prefer Nimbus, I can convert this to nimbus-jose-jwt; this keeps dependencies tiny.) + + static final class JwtUtil { + static String issueHs256Jwt(String issuer, String subject, String scope, long ttlSeconds, String secret) throws Exception { + long now = Instant.now().getEpochSecond(); + long exp = now + ttlSeconds; + + String headerJson = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String payloadJson = + "{" + + "\"iss\":\"" + jsonEscape(issuer) + "\"," + + "\"sub\":\"" + jsonEscape(subject) + "\"," + + "\"scope\":\"" + jsonEscape(scope) + "\"," + + "\"iat\":" + now + "," + + "\"exp\":" + exp + + "}"; + + String header = b64Url(headerJson.getBytes("UTF-8")); + String payload = b64Url(payloadJson.getBytes("UTF-8")); + String signingInput = header + "." + payload; + + byte[] sig = hmacSha256(signingInput.getBytes("UTF-8"), secret.getBytes("UTF-8")); + return signingInput + "." + b64Url(sig); + } + + static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data); + } + + static String b64Url(byte[] in) { + return java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(in); + } + + static String jsonEscape(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\""); + } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java new file mode 100644 index 000000000000..11a5f2b337d3 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + +import com.cloud.utils.Pair; + +public class PathUtil { + + public static Pair extractIdAndSubPath(final String path, final String baseRoute) { + + // baseRoute = "/api/datacenters" + if (!path.startsWith(baseRoute)) { + return null; + } + + // Remove base route + String rest = path.substring(baseRoute.length()); + + // Expect "" or "/{id}" or "/{id}/{sub}" + if (rest.isEmpty()) { + return null; // /api/datacenters (no id) + } + + if (!rest.startsWith("/")) { + return null; + } + + rest = rest.substring(1); // remove leading '/' + + final String[] parts = rest.split("/", -1); + + if (parts.length == 1) { + // /api/datacenters/{id} + if (parts[0].isEmpty()) return null; + return new Pair<>(parts[0], null); + } + + if (parts.length == 2) { + // /api/datacenters/{id}/{subPath} + if (parts[0].isEmpty() || parts[1].isEmpty()) return null; + return new Pair<>(parts[0], parts[1]); + } + + // deeper paths not handled here + return null; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java index a56dde4c75e3..46b3a993aa72 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.dataformat.xml.XmlMapper; public class ResponseMapper { @@ -36,6 +37,7 @@ public ResponseMapper() { private static void configure(final ObjectMapper mapper) { mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); // If you ever add enums etc: // mapper.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); // mapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java index a40ebc860a3c..7dcdc3e647fc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java @@ -24,8 +24,11 @@ import org.apache.cloudstack.veeam.api.dto.Fault; import org.apache.cloudstack.veeam.api.response.FaultResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public final class ResponseWriter { + private static final Logger LOGGER = LogManager.getLogger(ResponseWriter.class); private final ResponseMapper mapper; @@ -62,6 +65,8 @@ public void write(final HttpServletResponse resp, final int status, final Object return; } + LOGGER.info("Writing response: {}\n{}", status, payload); + resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); resp.setHeader("Content-Type", contentType); resp.getWriter().write(payload); diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 0d1d35730312..6e75d8384386 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -33,6 +33,7 @@ + diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java index 14d24dbb6410..52642cf03707 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -30,8 +30,11 @@ public class ServerPropertiesUtil { private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + protected static final String PROPERTIES_FILE = "server.properties"; protected static final AtomicReference propertiesRef = new AtomicReference<>(); + protected static final String KEYSTORE_FILE = "https.keystore"; + protected static final String KEYSTORE_PASSWORD = "https.keystore.password"; public static String getProperty(String name) { Properties props = propertiesRef.get(); @@ -55,4 +58,12 @@ public static String getProperty(String name) { } return tempProps.getProperty(name); } + + public static String getKeystoreFile() { + return getProperty(KEYSTORE_FILE); + } + + public static String getKeystorePassword() { + return getProperty(KEYSTORE_PASSWORD); + } } From a30eb280e5837c7d4837edb2c531b7fc1aed3414 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 22 Jan 2026 12:20:31 +0530 Subject: [PATCH 003/129] changes for discovery Signed-off-by: Abhishek Kumar --- .../veeam/api/DataCentersRouteHandler.java | 30 --- .../veeam/api/DisksRouteHandler.java | 111 ++++++++++ .../cloudstack/veeam/api/VmsRouteHandler.java | 56 +++-- .../converter/UserVmJoinVOToVmConverter.java | 10 +- .../VolumeJoinVOToDiskConverter.java | 198 ++++++++++++++++++ .../apache/cloudstack/veeam/api/dto/Bios.java | 2 + .../cloudstack/veeam/api/dto/BootMenu.java | 26 +++ .../apache/cloudstack/veeam/api/dto/Disk.java | 96 +++++++++ .../veeam/api/dto/DiskAttachment.java | 53 +++++ .../veeam/api/dto/DiskAttachments.java | 40 ++++ .../cloudstack/veeam/api/dto/Disks.java | 40 ++++ .../apache/cloudstack/veeam/api/dto/Vm.java | 2 +- .../spring-veeam-control-service-context.xml | 1 + .../cloud/api/query/dao/VolumeJoinDao.java | 2 + .../api/query/dao/VolumeJoinDaoImpl.java | 7 + 15 files changed, 617 insertions(+), 57 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index d297fe9b516d..c49f078121a5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -32,7 +32,6 @@ import org.apache.cloudstack.veeam.api.dto.DataCenters; import org.apache.cloudstack.veeam.api.dto.StorageDomain; import org.apache.cloudstack.veeam.api.dto.StorageDomains; -import org.apache.cloudstack.veeam.api.request.VmListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; @@ -106,18 +105,6 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - /** - * Matches /api/datacenters/{id} where {id} is a single path segment (no extra '/'). - * Returns id or null. - */ - private static String matchSinglePathParam(final String path, final String prefix) { - if (!path.startsWith(prefix)) return null; - final String rest = path.substring(prefix.length()); // after "/api/datacenters/" - if (rest.isEmpty()) return null; - if (rest.contains("/")) return null; // ensure only 1 segment - return rest; - } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = DataCenterVOToDataCenterConverter.toDCList(listDCs()); @@ -126,23 +113,6 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re io.getWriter().write(resp, 200, response, outFormat); } - private static VmListQuery fromRequest(final HttpServletRequest req) { - final VmListQuery q = new VmListQuery(); - q.setSearch(req.getParameter("search")); - q.setMax(parseIntOrNull(req.getParameter("max"))); - q.setPage(parseIntOrNull(req.getParameter("page"))); - return q; - } - - private static Integer parseIntOrNull(final String s) { - if (s == null || s.trim().isEmpty()) return null; - try { - return Integer.parseInt(s.trim()); - } catch (NumberFormatException e) { - return Integer.valueOf(-1); // will be rejected by validation above - } - } - protected List listDCs() { return dataCenterDao.listAll(); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java new file mode 100644 index 000000000000..ad7aed6455ba --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.api.dto.Disk; +import org.apache.cloudstack.veeam.api.dto.Disks; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.api.query.dao.VolumeJoinDao; +import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class DisksRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/disks"; + + @Inject + VolumeJoinDao volumeJoinDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/disks/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = VolumeJoinVOToDiskConverter.toDiskList(listDisks()); + final Disks response = new Disks(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listDisks() { + return volumeJoinDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final VolumeJoinVO volumeJoinVO = volumeJoinDao.findByUuid(id); + if (volumeJoinVO == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + Disk response = VolumeJoinVOToDiskConverter.toDisk(volumeJoinVO); + + io.getWriter().write(resp, 200, response, outFormat); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 23f626e326eb..62e7c67dfa72 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -28,6 +28,9 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; +import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.api.dto.DiskAttachment; +import org.apache.cloudstack.veeam.api.dto.DiskAttachments; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.request.VmListQuery; import org.apache.cloudstack.veeam.api.request.VmSearchExpr; @@ -36,9 +39,12 @@ import org.apache.cloudstack.veeam.api.response.VmCollectionResponse; import org.apache.cloudstack.veeam.api.response.VmEntityResponse; import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.dao.VolumeJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class VmsRouteHandler extends ManagerBase implements RouteHandler { @@ -50,6 +56,9 @@ public class VmsRouteHandler extends ManagerBase implements RouteHandler { @Inject UserVmJoinDao userVmJoinDao; + @Inject + VolumeJoinDao volumeJoinDao; + private VmSearchParser searchParser; @Override @@ -83,33 +92,24 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleGet(req, resp, outFormat, io); return; } - - // /api/vms/{id} - final String vmId = matchSinglePathParam(sanitizedPath, BASE_ROUTE + "/"); - if (vmId != null) { - if (!"GET".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET", outFormat); - return; + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/vms/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + if ("diskattachments".equals(idAndSubPath.second())) { + handleGetDisAttachmentsByVmId(idAndSubPath.first(), resp, outFormat, io); + return; + } } - handleGetById(vmId, resp, outFormat, io); - return; } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - /** - * Matches /api/vms/{id} where {id} is a single path segment (no extra '/'). - * Returns id or null. - */ - private static String matchSinglePathParam(final String path, final String prefix) { - if (!path.startsWith(prefix)) return null; - final String rest = path.substring(prefix.length()); // after "/api/vms/" - if (rest.isEmpty()) return null; - if (rest.contains("/")) return null; // ensure only 1 segment - return rest; - } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final VmListQuery q = fromRequest(req); @@ -182,4 +182,18 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } + + public void handleGetDisAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); + if (userVmJoinVO == null) { + io.notFound(resp, "VM not found: " + id, outFormat); + return; + } + List disks = VolumeJoinVOToDiskConverter.toDiskAttachmentList( + volumeJoinDao.listByInstanceId(userVmJoinVO.getId())); + DiskAttachments response = new DiskAttachments(disks); + + io.getWriter().write(resp, 200, response, outFormat); + } } \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 760ab5c758c4..4a8030149a8c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -66,28 +66,28 @@ public static Vm toVm(final UserVmJoinVO src) { } final Ref template = buildRef( basePath + ApiService.BASE_ROUTE, - "template", + "templates", src.getTemplateUuid() ); dst.template = template; dst.originalTemplate = template; dst.host = buildRef( basePath + ApiService.BASE_ROUTE, - "host", + "hosts", src.getHostUuid()); dst.cluster = buildRef( basePath + ApiService.BASE_ROUTE, - "cluster", + "clusters", src.getHostUuid()); dst.memory = src.getRamSize() * 1024L * 1024L; - dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); + dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), src.getCpu(), 1)); dst.os = new Os(); dst.os.type = src.getGuestOsId() % 2 == 0 ? "windows" : "linux"; dst.bios = new Bios(); - dst.bios.type = "legacy"; + dst.bios.type = "q35_secure_boot"; dst.type = "server"; dst.origin = "ovirt"; dst.actions = null;dst.link = List.of( diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java new file mode 100644 index 000000000000..55a25706a914 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.dto.Actions; +import org.apache.cloudstack.veeam.api.dto.Disk; +import org.apache.cloudstack.veeam.api.dto.DiskAttachment; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Ref; + +import com.cloud.api.ApiDBUtils; +import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.storage.Storage; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeStats; + +public class VolumeJoinVOToDiskConverter { + public static Disk toDisk(final VolumeJoinVO vol) { + final Disk disk = new Disk(); + final String apiBase = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; + + final String diskId = vol.getUuid(); + final String diskHref = apiBase + "/disks/" + diskId; + + disk.id = diskId; + disk.href = diskHref; + + // Names + disk.name = vol.getName(); + disk.alias = vol.getName(); + disk.description = ""; + + // Sizes (bytes) + final long size = vol.getSize(); + final long actualSize = vol.getVolumeStoreSize(); + + disk.provisionedSize = String.valueOf(size); + disk.actualSize = String.valueOf(actualSize); + disk.totalSize = String.valueOf(size); + VolumeStats vs = null; + if (List.of(Storage.ImageFormat.VHD, Storage.ImageFormat.QCOW2, Storage.ImageFormat.RAW).contains(vol.getFormat())) { + if (vol.getPath() != null) { + vs = ApiDBUtils.getVolumeStatistics(vol.getPath()); + } + } else if (vol.getFormat() == Storage.ImageFormat.OVA) { + if (vol.getChainInfo() != null) { + vs = ApiDBUtils.getVolumeStatistics(vol.getChainInfo()); + } + } + if (vs != null) { + disk.totalSize = String.valueOf(vs.getVirtualSize()); + disk.actualSize = String.valueOf(vs.getPhysicalSize()); + } + + // Disk format + disk.format = mapFormat(vol.getFormat()); + disk.qcowVersion = "qcow2_v3"; + + // Content & storage + disk.contentType = "data"; + disk.storageType = "image"; + disk.sparse = "true"; + disk.shareable = "false"; + + // Status + disk.status = mapStatus(vol.getState()); + + // Backup-related flags (safe defaults) + disk.backup = "none"; + disk.propagateErrors = "false"; + disk.wipeAfterDelete = "false"; + + // Image ID (best-effort) + disk.imageId = vol.getPath(); // acceptable placeholder + + // Disk profile (optional) + disk.diskProfile = Ref.of( + apiBase + "/diskprofiles/" + vol.getDiskOfferingId(), + String.valueOf(vol.getDiskOfferingId()) + ); + + // Storage domains + if (vol.getPoolUuid() != null) { + Disk.StorageDomains sds = new Disk.StorageDomains(); + sds.storageDomain = List.of( + Ref.of( + apiBase + "/storagedomains/" + vol.getPoolUuid(), + vol.getPoolUuid() + ) + ); + disk.storageDomains = sds; + } + + // Actions (Veeam checks presence, not behavior) + disk.actions = defaultDiskActions(diskHref); + + // Links + disk.link = List.of( + new Link("disksnapshots", diskHref + "/disksnapshots") + ); + + return disk; + } + + public static List toDiskList(final List srcList) { + return srcList.stream() + .map(VolumeJoinVOToDiskConverter::toDisk) + .collect(Collectors.toList()); + } + + public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { + final DiskAttachment da = new DiskAttachment(); + final String apiBase = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; + + final String diskAttachmentId = vol.getUuid(); + final String diskAttachmentHref = apiBase + "/diskattachments/" + diskAttachmentId; + + da.id = diskAttachmentId; + da.href = diskAttachmentHref; + + // Links + da.disk = Ref.of( + apiBase + "/disks/" + vol.getUuid(), + vol.getUuid() + ); + da.vm = Ref.of( + apiBase + "/vms/" + vol.getVmUuid(), + vol.getVmUuid() + ); + + // Properties + da.active = "true"; + da.bootable = "false"; + da.iface = "virtio_scsi"; + da.logicalName = vol.getName(); + da.readOnly = "false"; + da.passDiscard = "false"; + + return da; + } + + public static List toDiskAttachmentList(final List srcList) { + return srcList.stream() + .map(VolumeJoinVOToDiskConverter::toDiskAttachment) + .collect(Collectors.toList()); + } + + private static String mapFormat(final Storage.ImageFormat format) { + if (format == null) { + return "cow"; + } + switch (format) { + case RAW: + return "raw"; + case QCOW2: + default: + return "cow"; + } + } + + private static String mapStatus(final Volume.State state) { + if (state == null) { + return "ok"; + } + switch (state.name().toLowerCase()) { + case "ready": + case "allocated": + return "ok"; + default: + return "locked"; + } + } + + private static Actions defaultDiskActions(final String diskHref) { + return new Actions(Collections.emptyList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java index f1de8cf3a5ad..fa9e46ba87cf 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java @@ -23,6 +23,8 @@ public final class Bios { public String type; // "uefi" or "bios" or whatever mapping you choose + public BootMenu bootMenu = new BootMenu(); + public Bios() {} public Bios(final String type) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java new file mode 100644 index 000000000000..714b256596ad --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class BootMenu { + + public String enabled = "false"; +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java new file mode 100644 index 000000000000..812501f5615d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "disk") +public final class Disk { + + @JsonProperty("actual_size") + public String actualSize; + + public String alias; + public String backup; + + @JsonProperty("content_type") + public String contentType; + + public String format; + + @JsonProperty("image_id") + public String imageId; + + @JsonProperty("propagate_errors") + public String propagateErrors; + + @JsonProperty("provisioned_size") + public String provisionedSize; + + @JsonProperty("qcow_version") + public String qcowVersion; + + public String shareable; + public String sparse; + public String status; + + @JsonProperty("storage_type") + public String storageType; + + @JsonProperty("total_size") + public String totalSize; + + @JsonProperty("wipe_after_delete") + public String wipeAfterDelete; + + @JsonProperty("disk_profile") + public Ref diskProfile; + + public Ref quota; + + @JsonProperty("storage_domains") + public StorageDomains storageDomains; + + public Actions actions; + + public String name; + public String description; + + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + public String href; + public String id; + + public Disk() {} + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlRootElement(localName = "storage_domains") + public static final class StorageDomains { + @JsonProperty("storage_domain") + @JacksonXmlElementWrapper(useWrapping = false) + public List storageDomain; + public StorageDomains() {} + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java new file mode 100644 index 000000000000..ca041e993f5e --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "disk_attachment") +public final class DiskAttachment { + + public String active; + public String bootable; + + @JsonProperty("interface") + public String iface; // virtio_scsi etc + + @JsonProperty("logical_name") + public String logicalName; + + @JsonProperty("pass_discard") + public String passDiscard; + + @JsonProperty("read_only") + public String readOnly; + + @JsonProperty("uses_scsi_reservation") + public String usesScsiReservation; + + public Ref disk; + public Ref vm; + + public String href; + public String id; + + public DiskAttachment() {} +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java new file mode 100644 index 000000000000..deebb9d310aa --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "disk_attachments") +public final class DiskAttachments { + + @JsonProperty("disk_attachment") + @JacksonXmlElementWrapper(useWrapping = false) + public List diskAttachment; + + public DiskAttachments() {} + + public DiskAttachments(final List diskAttachment) { + this.diskAttachment = diskAttachment; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java new file mode 100644 index 000000000000..302ff3adfd85 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "disks") +public final class Disks { + + @JsonProperty("disk") + @JacksonXmlElementWrapper(useWrapping = false) + public List disk; + + public Disks() {} + + public Disks(final List disk) { + this.disk = disk; + } +} \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 4bba580a9712..5a21f84c4ae5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -61,7 +61,7 @@ public final class Vm { public Os os; public Bios bios; - public boolean stateless; // true|false + public String stateless = "false"; // true|false public String type; // "server" public String origin; // "ovirt" diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 6e75d8384386..0c553d8e5536 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -34,6 +34,7 @@ + diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java index ebcf0bca391e..87485e86fc9e 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java @@ -34,4 +34,6 @@ public interface VolumeJoinDao extends GenericDao { List newVolumeView(Volume vol); List searchByIds(Long... ids); + + List listByInstanceId(long instanceId); } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 4f5d984c969a..9361abef6043 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -372,4 +372,11 @@ public List searchByIds(Long... volIds) { return uvList; } + @Override + public List listByInstanceId(long instanceId) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("vmId", SearchCriteria.Op.EQ, instanceId); + return search(sc, null); + } + } From f52b114c8db92090ec6f4a7192d9c60c8007e2e9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 22 Jan 2026 16:43:05 +0530 Subject: [PATCH 004/129] changes Signed-off-by: Abhishek Kumar --- .../veeam/api/ClustersRouteHandler.java | 123 ++++++++ .../cloudstack/veeam/api/VmsRouteHandler.java | 17 +- .../ClusterVOToClusterConverter.java | 170 +++++++++++ .../DataCenterVOToDataCenterConverter.java | 2 +- .../StoreVOToStorageDomainConverter.java | 4 +- .../converter/UserVmJoinVOToVmConverter.java | 42 ++- .../VolumeJoinVOToDiskConverter.java | 11 +- .../cloudstack/veeam/api/dto/Cluster.java | 280 ++++++++++++++++++ .../cloudstack/veeam/api/dto/Clusters.java | 40 +++ .../spring-veeam-control-service-context.xml | 1 + 10 files changed, 669 insertions(+), 21 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java new file mode 100644 index 000000000000..bb14b6144797 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.ClusterVOToClusterConverter; +import org.apache.cloudstack.veeam.api.dto.Cluster; +import org.apache.cloudstack.veeam.api.dto.Clusters; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class ClustersRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/clusters"; + + @Inject + ClusterDao clusterDao; + + @Inject + DataCenterDao dataCenterDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/disks/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = ClusterVOToClusterConverter.toClusterList(listClusters(), this::getZoneById); + final Clusters response = new Clusters(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listClusters() { + return clusterDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final ClusterVO vo = clusterDao.findByUuid(id); + if (vo == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + Cluster response = ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); + + io.getWriter().write(resp, 200, response, outFormat); + } + + private DataCenterVO getZoneById(Long zoneId) { + if (zoneId == null) { + return null; + } + return dataCenterDao.findById(zoneId); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 62e7c67dfa72..dc92f58715f5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -41,8 +41,10 @@ import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.dao.VolumeJoinDao; +import com.cloud.api.query.vo.HostJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; @@ -56,6 +58,9 @@ public class VmsRouteHandler extends ManagerBase implements RouteHandler { @Inject UserVmJoinDao userVmJoinDao; + @Inject + HostJoinDao hostJoinDao; + @Inject VolumeJoinDao volumeJoinDao; @@ -143,7 +148,7 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re return; } - final List result = UserVmJoinVOToVmConverter.toVmList(listUserVms()); + final List result = UserVmJoinVOToVmConverter.toVmList(listUserVms(), this::getHostById); final VmCollectionResponse response = new VmCollectionResponse(result); io.getWriter().write(resp, 200, response, outFormat); @@ -178,7 +183,7 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.notFound(resp, "VM not found: " + id, outFormat); return; } - VmEntityResponse response = new VmEntityResponse(UserVmJoinVOToVmConverter.toVm(userVmJoinVO)); + VmEntityResponse response = new VmEntityResponse(UserVmJoinVOToVmConverter.toVm(userVmJoinVO, this::getHostById)); io.getWriter().write(resp, 200, response, outFormat); } @@ -196,4 +201,12 @@ public void handleGetDisAttachmentsByVmId(final String id, final HttpServletResp io.getWriter().write(resp, 200, response, outFormat); } + + private HostJoinVO getHostById(Long hostId) { + if (hostId == null) { + return null; + } + return hostJoinDao.findById(hostId); + } + } \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java new file mode 100644 index 000000000000..54176d4004ae --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ClustersRouteHandler; +import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Actions; +import org.apache.cloudstack.veeam.api.dto.Cluster; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Ref; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; + +public class ClusterVOToClusterConverter { + public static Cluster toCluster(final ClusterVO vo, final Function dataCenterResolver) { + final Cluster c = new Cluster(); + final String basePath = VeeamControlService.ContextPath.value(); + + // NOTE: oVirt uses UUIDs. If your ClusterVO id is numeric, generate a stable UUID: + // - Prefer: store a UUID in details table and reuse it + // - Fallback: name-based UUID from "cluster:" + final String clusterId = vo.getUuid(); + c.id = clusterId; + c.href = basePath + ClustersRouteHandler.BASE_ROUTE + "/" + clusterId; + + c.name = vo.getName(); + c.description = vo.getName(); + c.comment = ""; + + // --- sensible defaults (match your sample) + c.ballooningEnabled = "true"; + c.biosType = "q35_ovmf"; // or "q35_secure_boot" if you want to align with VM BIOS you saw + c.fipsMode = "disabled"; + c.firewallType = "firewalld"; + c.glusterService = "false"; + c.haReservation = "false"; + c.switchType = "legacy"; + c.threadsAsCores = "false"; + c.trustedService = "false"; + c.tunnelMigration = "false"; + c.upgradeInProgress = "false"; + c.upgradePercentComplete = "0"; + c.virtService = "true"; + c.vncEncryption = "false"; + c.logMaxMemoryUsedThreshold = "95"; + c.logMaxMemoryUsedThresholdType = "percentage"; + + // --- cpu (best-effort defaults) + final Cluster.ClusterCpu cpu = new Cluster.ClusterCpu(); + cpu.architecture = "x86_64"; + cpu.type = "x86_64"; // replace if you can detect host cpu model + c.cpu = cpu; + + // --- version (ovirt engine version; keep fixed unless you want to expose something else) + final Cluster.Version ver = new Cluster.Version(); + ver.major = "4"; + ver.minor = "8"; + c.version = ver; + + // --- ksm / memory policy (defaults) + c.ksm = new Cluster.Ksm(); + c.ksm.enabled = "true"; + c.ksm.mergeAcrossNodes = "true"; + + c.memoryPolicy = new Cluster.MemoryPolicy(); + c.memoryPolicy.overCommit = new Cluster.OverCommit(); + c.memoryPolicy.overCommit.percent = "100"; + c.memoryPolicy.transparentHugepages = new Cluster.TransparentHugepages(); + c.memoryPolicy.transparentHugepages.enabled = "true"; + + // --- migration defaults + c.migration = new Cluster.Migration(); + c.migration.autoConverge = "inherit"; + c.migration.bandwidth = new Cluster.Bandwidth(); + c.migration.bandwidth.assignmentMethod = "auto"; + c.migration.compressed = "inherit"; + c.migration.encrypted = "inherit"; + c.migration.parallelMigrationsPolicy = "disabled"; + // policy ref (dummy but valid shape) + c.migration.policy = Ref.of(basePath + "/migrationpolicies/" + stableUuid("migrationpolicy:default"), + stableUuid("migrationpolicy:default") + ); + + // --- rng sources + c.requiredRngSources = new Cluster.RequiredRngSources(); + c.requiredRngSources.requiredRngSource = Collections.singletonList("urandom"); + + // --- error handling + c.errorHandling = new Cluster.ErrorHandling(); + c.errorHandling.onError = "migrate"; + + // --- fencing policy defaults + c.fencingPolicy = new Cluster.FencingPolicy(); + c.fencingPolicy.enabled = "true"; + c.fencingPolicy.skipIfConnectivityBroken = new Cluster.SkipIfConnectivityBroken(); + c.fencingPolicy.skipIfConnectivityBroken.enabled = "false"; + c.fencingPolicy.skipIfConnectivityBroken.threshold = "50"; + c.fencingPolicy.skipIfGlusterBricksUp = "false"; + c.fencingPolicy.skipIfGlusterQuorumNotMet = "false"; + c.fencingPolicy.skipIfSdActive = new Cluster.SkipIfSdActive(); + c.fencingPolicy.skipIfSdActive.enabled = "false"; + + // --- scheduling policy props (optional; dummy ok) + c.customSchedulingPolicyProperties = new Cluster.CustomSchedulingPolicyProperties(); + final Cluster.Property p1 = new Cluster.Property(); p1.name = "HighUtilization"; p1.value = "80"; + final Cluster.Property p2 = new Cluster.Property(); p2.name = "CpuOverCommitDurationMinutes"; p2.value = "2"; + c.customSchedulingPolicyProperties.property = List.of(p1, p2); + + // --- data_center ref mapping (CloudStack cluster -> pod -> zone) + if (dataCenterResolver != null) { + final DataCenterVO zone = dataCenterResolver.apply(vo.getDataCenterId()); + if (zone != null) { + c.dataCenter = Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + zone.getUuid(), zone.getUuid()); + } + } + + // --- mac pool & scheduling policy refs (dummy but consistent) + c.macPool = Ref.of(basePath + "/macpools/" + stableUuid("macpool:default"), + stableUuid("macpool:default")); + c.schedulingPolicy = Ref.of(basePath + "/schedulingpolicies/" + stableUuid("schedpolicy:default"), + stableUuid("schedpolicy:default")); + + // --- actions.links (can be omitted; but Veeam sometimes expects actions to exist) + final Actions actions = new Actions(); + actions.link = Collections.emptyList(); + c.actions = actions; + + // --- related links (optional) + c.link = List.of( + new Link("networks", c.href + "/networks") + ); + + return c; + } + + public static List toClusterList(final List voList, + final Function dataCenterResolver) { + return voList.stream() + .map(vo -> toCluster(vo, dataCenterResolver)) + .collect(Collectors.toList()); + } + + private static String stableUuid(final String key) { + // deterministic UUID, so the same ClusterVO maps to same "ovirt id" every time + return UUID.nameUUIDFromBytes(key.getBytes()).toString(); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java index 395bb233ea53..c39b91a9684c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java @@ -36,7 +36,7 @@ public class DataCenterVOToDataCenterConverter { public static DataCenter toDataCenter(final DataCenterVO zone) { final String id = zone.getUuid(); final String basePath = VeeamControlService.ContextPath.value(); - final String href = basePath + DataCentersRouteHandler.BASE_ROUTE + "/datacenters/" + id; + final String href = basePath + DataCentersRouteHandler.BASE_ROUTE + DataCentersRouteHandler.BASE_ROUTE + "/" + id; final DataCenter dc = new DataCenter(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java index 071ebc92c142..f974826ce40e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java @@ -77,7 +77,7 @@ public static StorageDomain toStorageDomain(final StoragePoolJoinVO pool) { // dc attachment String dcId = pool.getZoneUuid(); DataCenter dc = new DataCenter(); - dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + dcId); + dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + "/" + dcId); dc.id = dcId; sd.dataCenters = new DataCenters(List.of(dc)); @@ -132,7 +132,7 @@ public static StorageDomain toStorageDomain(final ImageStoreJoinVO store) { // Optionally include dc attachment (your first object had it; second didn’t) String dcId = store.getZoneUuid(); DataCenter dc = new DataCenter(); - dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + dcId); + dc.href = href(basePath, DataCentersRouteHandler.BASE_ROUTE + "/" + dcId); dc.id = dcId; sd.dataCenters = new DataCenters(List.of(dc)); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 4a8030149a8c..8fb2578a0286 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.cloudstack.veeam.VeeamControlService; @@ -34,6 +35,7 @@ import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.commons.lang3.StringUtils; +import com.cloud.api.query.vo.HostJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.vm.VirtualMachine; @@ -47,7 +49,7 @@ private UserVmJoinVOToVmConverter() { * * @param src UserVmJoinVO */ - public static Vm toVm(final UserVmJoinVO src) { + public static Vm toVm(final UserVmJoinVO src, final Function hostResolver) { if (src == null) { return null; } @@ -71,14 +73,32 @@ public static Vm toVm(final UserVmJoinVO src) { ); dst.template = template; dst.originalTemplate = template; - dst.host = buildRef( - basePath + ApiService.BASE_ROUTE, - "hosts", - src.getHostUuid()); - dst.cluster = buildRef( - basePath + ApiService.BASE_ROUTE, - "clusters", - src.getHostUuid()); + if (StringUtils.isNotBlank(src.getHostUuid())) { + dst.host = buildRef( + basePath + ApiService.BASE_ROUTE, + "hosts", + src.getHostUuid()); + + } + if (hostResolver != null) { + HostJoinVO hostVo = hostResolver.apply(src.getHostId() == null ? src.getLastHostId() : src.getHostId()); + if (hostVo != null) { + dst.host = buildRef( + basePath + ApiService.BASE_ROUTE, + "hosts", + hostVo.getUuid()); + dst.cluster = buildRef( + basePath + ApiService.BASE_ROUTE, + "clusters", + hostVo.getClusterUuid()); + } + } + Long hostId = src.getHostId() != null ? src.getHostId() : src.getLastHostId(); + if (hostId != null) { + // I want to get Host data from hostJoinDao but this is a static method without dao access. + + } + dst.memory = src.getRamSize() * 1024L * 1024L; dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), src.getCpu(), 1)); @@ -102,9 +122,9 @@ public static Vm toVm(final UserVmJoinVO src) { return dst; } - public static List toVmList(final List srcList) { + public static List toVmList(final List srcList, final Function hostResolver) { return srcList.stream() - .map(UserVmJoinVOToVmConverter::toVm) + .map(v -> toVm(v, hostResolver)) .collect(Collectors.toList()); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 55a25706a914..3b2305f52186 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.DisksRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; @@ -38,10 +39,10 @@ public class VolumeJoinVOToDiskConverter { public static Disk toDisk(final VolumeJoinVO vol) { final Disk disk = new Disk(); - final String apiBase = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; + final String basePath = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; final String diskId = vol.getUuid(); - final String diskHref = apiBase + "/disks/" + diskId; + final String diskHref = basePath + DisksRouteHandler.BASE_ROUTE + "/" + diskId; disk.id = diskId; disk.href = diskHref; @@ -49,7 +50,7 @@ public static Disk toDisk(final VolumeJoinVO vol) { // Names disk.name = vol.getName(); disk.alias = vol.getName(); - disk.description = ""; + disk.description = vol.getName(); // Sizes (bytes) final long size = vol.getSize(); @@ -96,7 +97,7 @@ public static Disk toDisk(final VolumeJoinVO vol) { // Disk profile (optional) disk.diskProfile = Ref.of( - apiBase + "/diskprofiles/" + vol.getDiskOfferingId(), + basePath + "/diskprofiles/" + vol.getDiskOfferingId(), String.valueOf(vol.getDiskOfferingId()) ); @@ -105,7 +106,7 @@ public static Disk toDisk(final VolumeJoinVO vol) { Disk.StorageDomains sds = new Disk.StorageDomains(); sds.storageDomain = List.of( Ref.of( - apiBase + "/storagedomains/" + vol.getPoolUuid(), + basePath + "/storagedomains/" + vol.getPoolUuid(), vol.getPoolUuid() ) ); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java new file mode 100644 index 000000000000..cdd4a18e2cc9 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "cluster") +public final class Cluster { + + // --- common identity + public String href; + public String id; + public String name; + public String description; + public String comment; + + // --- oVirt-ish knobs (strings in oVirt JSON) + @JsonProperty("ballooning_enabled") + @JacksonXmlProperty(localName = "ballooning_enabled") + public String ballooningEnabled; // "true"/"false" + + @JsonProperty("bios_type") + @JacksonXmlProperty(localName = "bios_type") + public String biosType; // e.g. "q35_ovmf" + + public ClusterCpu cpu; + + @JsonProperty("custom_scheduling_policy_properties") + @JacksonXmlProperty(localName = "custom_scheduling_policy_properties") + public CustomSchedulingPolicyProperties customSchedulingPolicyProperties; + + @JsonProperty("error_handling") + @JacksonXmlProperty(localName = "error_handling") + public ErrorHandling errorHandling; + + @JsonProperty("fencing_policy") + @JacksonXmlProperty(localName = "fencing_policy") + public FencingPolicy fencingPolicy; + + @JsonProperty("fips_mode") + @JacksonXmlProperty(localName = "fips_mode") + public String fipsMode; // "disabled" + + @JsonProperty("firewall_type") + @JacksonXmlProperty(localName = "firewall_type") + public String firewallType; // "firewalld" + + @JsonProperty("gluster_service") + @JacksonXmlProperty(localName = "gluster_service") + public String glusterService; + + @JsonProperty("ha_reservation") + @JacksonXmlProperty(localName = "ha_reservation") + public String haReservation; + + public Ksm ksm; + + @JsonProperty("log_max_memory_used_threshold") + @JacksonXmlProperty(localName = "log_max_memory_used_threshold") + public String logMaxMemoryUsedThreshold; + + @JsonProperty("log_max_memory_used_threshold_type") + @JacksonXmlProperty(localName = "log_max_memory_used_threshold_type") + public String logMaxMemoryUsedThresholdType; + + @JsonProperty("memory_policy") + @JacksonXmlProperty(localName = "memory_policy") + public MemoryPolicy memoryPolicy; + + public Migration migration; + + @JsonProperty("required_rng_sources") + @JacksonXmlProperty(localName = "required_rng_sources") + public RequiredRngSources requiredRngSources; + + @JsonProperty("switch_type") + @JacksonXmlProperty(localName = "switch_type") + public String switchType; + + @JsonProperty("threads_as_cores") + @JacksonXmlProperty(localName = "threads_as_cores") + public String threadsAsCores; + + @JsonProperty("trusted_service") + @JacksonXmlProperty(localName = "trusted_service") + public String trustedService; + + @JsonProperty("tunnel_migration") + @JacksonXmlProperty(localName = "tunnel_migration") + public String tunnelMigration; + + @JsonProperty("upgrade_in_progress") + @JacksonXmlProperty(localName = "upgrade_in_progress") + public String upgradeInProgress; + + @JsonProperty("upgrade_percent_complete") + @JacksonXmlProperty(localName = "upgrade_percent_complete") + public String upgradePercentComplete; + + public Version version; + + @JsonProperty("virt_service") + @JacksonXmlProperty(localName = "virt_service") + public String virtService; + + @JsonProperty("vnc_encryption") + @JacksonXmlProperty(localName = "vnc_encryption") + public String vncEncryption; + + // --- references + @JsonProperty("data_center") + @JacksonXmlProperty(localName = "data_center") + public Ref dataCenter; + + @JsonProperty("mac_pool") + @JacksonXmlProperty(localName = "mac_pool") + public Ref macPool; + + @JsonProperty("scheduling_policy") + @JacksonXmlProperty(localName = "scheduling_policy") + public Ref schedulingPolicy; + + // --- actions + links + public Actions actions; + + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + public Cluster() {} + + // ===== nested DTOs ===== + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class ClusterCpu { + public String architecture; + public String type; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class CustomSchedulingPolicyProperties { + @JacksonXmlElementWrapper(useWrapping = false) + @JsonProperty("property") + public List property; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Property { + public String name; + public String value; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class ErrorHandling { + @JsonProperty("on_error") + @JacksonXmlProperty(localName = "on_error") + public String onError; // "migrate" + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class FencingPolicy { + public String enabled; + + @JsonProperty("skip_if_connectivity_broken") + @JacksonXmlProperty(localName = "skip_if_connectivity_broken") + public SkipIfConnectivityBroken skipIfConnectivityBroken; + + @JsonProperty("skip_if_gluster_bricks_up") + @JacksonXmlProperty(localName = "skip_if_gluster_bricks_up") + public String skipIfGlusterBricksUp; + + @JsonProperty("skip_if_gluster_quorum_not_met") + @JacksonXmlProperty(localName = "skip_if_gluster_quorum_not_met") + public String skipIfGlusterQuorumNotMet; + + @JsonProperty("skip_if_sd_active") + @JacksonXmlProperty(localName = "skip_if_sd_active") + public SkipIfSdActive skipIfSdActive; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class SkipIfConnectivityBroken { + public String enabled; + public String threshold; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class SkipIfSdActive { + public String enabled; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Ksm { + public String enabled; + + @JsonProperty("merge_across_nodes") + @JacksonXmlProperty(localName = "merge_across_nodes") + public String mergeAcrossNodes; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class MemoryPolicy { + @JsonProperty("over_commit") + @JacksonXmlProperty(localName = "over_commit") + public OverCommit overCommit; + + @JsonProperty("transparent_hugepages") + @JacksonXmlProperty(localName = "transparent_hugepages") + public TransparentHugepages transparentHugepages; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class OverCommit { + public String percent; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class TransparentHugepages { + public String enabled; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Migration { + @JsonProperty("auto_converge") + @JacksonXmlProperty(localName = "auto_converge") + public String autoConverge; + + public Bandwidth bandwidth; + + public String compressed; + public String encrypted; + + @JsonProperty("parallel_migrations_policy") + @JacksonXmlProperty(localName = "parallel_migrations_policy") + public String parallelMigrationsPolicy; + + public Ref policy; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Bandwidth { + @JsonProperty("assignment_method") + @JacksonXmlProperty(localName = "assignment_method") + public String assignmentMethod; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class RequiredRngSources { + @JsonProperty("required_rng_source") + @JacksonXmlElementWrapper(useWrapping = false) + public List requiredRngSource; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Version { + public String major; + public String minor; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java new file mode 100644 index 000000000000..67eca4c989c2 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "clusters") +public final class Clusters { + + @JsonProperty("cluster") + @JacksonXmlElementWrapper(useWrapping = false) + public List cluster; + + public Clusters() {} + + public Clusters(final List cluster) { + this.cluster = cluster; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 0c553d8e5536..1ed843cbb46d 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -34,6 +34,7 @@ + From 27844684c55fe1c01d701cfbe562d644aa221858 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 23 Jan 2026 16:57:28 +0530 Subject: [PATCH 005/129] changes Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/api/ApiService.java | 8 +- .../veeam/api/ClustersRouteHandler.java | 12 +- .../veeam/api/DataCentersRouteHandler.java | 52 ++++-- .../veeam/api/HostsRouteHandler.java | 111 ++++++++++++ .../veeam/api/NetworksRouteHandler.java | 123 +++++++++++++ .../cloudstack/veeam/api/VmsRouteHandler.java | 1 - .../veeam/api/VnicProfilesRouteHandler.java | 123 +++++++++++++ .../ClusterVOToClusterConverter.java | 12 +- ...ataCenterJoinVOToDataCenterConverter.java} | 10 +- .../converter/HostJoinVOToHostConverter.java | 113 ++++++++++++ .../NetworkVOToNetworkConverter.java | 80 +++++++++ .../NetworkVOToVnicProfileConverter.java | 65 +++++++ .../apache/cloudstack/veeam/api/dto/Api.java | 2 +- .../api/dto/{Summary.java => ApiSummary.java} | 4 +- .../cloudstack/veeam/api/dto/Certificate.java | 36 ++++ .../apache/cloudstack/veeam/api/dto/Cpu.java | 11 ++ .../veeam/api/dto/HardwareInformation.java | 51 ++++++ .../apache/cloudstack/veeam/api/dto/Host.java | 169 ++++++++++++++++++ .../cloudstack/veeam/api/dto/HostSummary.java | 40 +++++ .../cloudstack/veeam/api/dto/Hosts.java | 33 ++++ .../cloudstack/veeam/api/dto/Network.java | 85 +++++++++ .../veeam/api/dto/NetworkUsages.java | 42 +++++ .../cloudstack/veeam/api/dto/Networks.java | 33 ++++ .../cloudstack/veeam/api/dto/OsVersion.java | 41 +++++ .../cloudstack/veeam/api/dto/VnicProfile.java | 99 ++++++++++ .../veeam/api/dto/VnicProfiles.java | 49 +++++ .../veeam/filter/BearerOrBasicAuthFilter.java | 2 +- .../cloudstack/veeam/sso/SsoService.java | 4 +- .../spring-veeam-control-service-context.xml | 5 +- 29 files changed, 1377 insertions(+), 39 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/{DataCenterVOToDataCenterConverter.java => DataCenterJoinVOToDataCenterConverter.java} (91%) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/{Summary.java => ApiSummary.java} (95%) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index bb37c300d846..24a9dbb730ee 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.veeam.api; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; @@ -37,11 +38,12 @@ import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.SpecialObjectRef; import org.apache.cloudstack.veeam.api.dto.SpecialObjects; -import org.apache.cloudstack.veeam.api.dto.Summary; +import org.apache.cloudstack.veeam.api.dto.ApiSummary; import org.apache.cloudstack.veeam.api.dto.SummaryCount; import org.apache.cloudstack.veeam.api.dto.Version; import org.apache.cloudstack.veeam.utils.Negotiation; +import com.cloud.utils.UuidUtils; import com.cloud.utils.component.ManagerBase; public class ApiService extends ManagerBase implements RouteHandler { @@ -99,7 +101,7 @@ private static Api createDummyApi(String basePath) { /* ---------------- Product info ---------------- */ ProductInfo productInfo = new ProductInfo(); - productInfo.instanceId = UUID.randomUUID().toString(); + productInfo.instanceId = UuidUtils.nameUUIDFromBytes(VeeamControlService.BindAddress.value().getBytes(StandardCharsets.UTF_8)).toString(); productInfo.name = "oVirt Engine"; Version version = new Version(); @@ -125,7 +127,7 @@ private static Api createDummyApi(String basePath) { api.specialObjects = specialObjects; /* ---------------- Summary ---------------- */ - Summary summary = new Summary(); + ApiSummary summary = new ApiSummary(); summary.hosts = new SummaryCount(1, 1); summary.storageDomains = new SummaryCount(1, 2); summary.users = new SummaryCount(1, 1); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index bb14b6144797..6459ad06f827 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -32,10 +32,10 @@ import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import com.cloud.api.query.dao.DataCenterJoinDao; +import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; -import com.cloud.dc.dao.DataCenterDao; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; @@ -46,7 +46,7 @@ public class ClustersRouteHandler extends ManagerBase implements RouteHandler { ClusterDao clusterDao; @Inject - DataCenterDao dataCenterDao; + DataCenterJoinDao dataCenterJoinDao; @Override public boolean start() { @@ -78,7 +78,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); if (idAndSubPath != null) { - // /api/disks/{id} + // /api/clusters/{id} if (idAndSubPath.first() != null) { if (idAndSubPath.second() == null) { handleGetById(idAndSubPath.first(), resp, outFormat, io); @@ -114,10 +114,10 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } - private DataCenterVO getZoneById(Long zoneId) { + private DataCenterJoinVO getZoneById(Long zoneId) { if (zoneId == null) { return null; } - return dataCenterDao.findById(zoneId); + return dataCenterJoinDao.findById(zoneId); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index c49f078121a5..459b076fefe7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -26,21 +26,26 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.DataCenterVOToDataCenterConverter; +import org.apache.cloudstack.veeam.api.converter.DataCenterJoinVOToDataCenterConverter; +import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; import org.apache.cloudstack.veeam.api.dto.DataCenter; import org.apache.cloudstack.veeam.api.dto.DataCenters; +import org.apache.cloudstack.veeam.api.dto.Network; +import org.apache.cloudstack.veeam.api.dto.Networks; import org.apache.cloudstack.veeam.api.dto.StorageDomain; import org.apache.cloudstack.veeam.api.dto.StorageDomains; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.dao.ImageStoreJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; +import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.api.query.vo.ImageStoreJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.dao.DataCenterDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; @@ -51,7 +56,7 @@ public class DataCentersRouteHandler extends ManagerBase implements RouteHandler private static final int DEFAULT_PAGE = 1; @Inject - DataCenterDao dataCenterDao; + DataCenterJoinDao dataCenterJoinDao; @Inject StoragePoolJoinDao storagePoolJoinDao; @@ -59,6 +64,9 @@ public class DataCentersRouteHandler extends ManagerBase implements RouteHandler @Inject ImageStoreJoinDao imageStoreJoinDao; + @Inject + NetworkDao networkDao; + @Override public boolean start() { return true; @@ -99,6 +107,10 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleGetStorageDomainsByDcId(idAndSubPath.first(), resp, outFormat, io); return; } + if ("networks".equals(idAndSubPath.second())) { + handleGetNetworksByDcId(idAndSubPath.first(), resp, outFormat, io); + return; + } } } @@ -107,24 +119,24 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = DataCenterVOToDataCenterConverter.toDCList(listDCs()); + final List result = DataCenterJoinVOToDataCenterConverter.toDCList(listDCs()); final DataCenters response = new DataCenters(result); io.getWriter().write(resp, 200, response, outFormat); } - protected List listDCs() { - return dataCenterDao.listAll(); + protected List listDCs() { + return dataCenterJoinDao.listAll(); } public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(id); + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); if (dataCenterVO == null) { io.notFound(resp, "DataCenter not found: " + id, outFormat); return; } - DataCenter response = DataCenterVOToDataCenterConverter.toDataCenter(dataCenterVO); + DataCenter response = DataCenterJoinVOToDataCenterConverter.toDataCenter(dataCenterVO); io.getWriter().write(resp, 200, response, outFormat); } @@ -137,9 +149,13 @@ protected List listImageStoresByDcId(final long dcId) { return imageStoreJoinDao.listAll(); } + protected List listNetworksByDcId(final long dcId) { + return networkDao.listAll(); + } + public void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { - final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(id); + final VeeamControlServlet io) throws IOException { + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); if (dataCenterVO == null) { io.notFound(resp, "DataCenter not found: " + id, outFormat); return; @@ -151,4 +167,18 @@ public void handleGetStorageDomainsByDcId(final String id, final HttpServletResp io.getWriter().write(resp, 200, response, outFormat); } + + public void handleGetNetworksByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); + if (dataCenterVO == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + List networks = NetworkVOToNetworkConverter.toNetworkList(listNetworksByDcId(dataCenterVO.getId()), (dcId) -> dataCenterVO); + + Networks response = new Networks(networks); + + io.getWriter().write(resp, 200, response, outFormat); + } } \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java new file mode 100644 index 000000000000..b33fa9bda9ce --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.HostJoinVOToHostConverter; +import org.apache.cloudstack.veeam.api.dto.Host; +import org.apache.cloudstack.veeam.api.dto.Hosts; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.api.query.dao.HostJoinDao; +import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class HostsRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/hosts"; + + @Inject + HostJoinDao hostJoinDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/hosts/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = HostJoinVOToHostConverter.toHostList(listHosts()); + final Hosts response = new Hosts(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listHosts() { + return hostJoinDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final HostJoinVO vo = hostJoinDao.findByUuid(id); + if (vo == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + Host response = HostJoinVOToHostConverter.toHost(vo); + + io.getWriter().write(resp, 200, response, outFormat); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java new file mode 100644 index 000000000000..c3bab348f4e4 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; +import org.apache.cloudstack.veeam.api.dto.Network; +import org.apache.cloudstack.veeam.api.dto.Networks; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.api.query.dao.DataCenterJoinDao; +import com.cloud.api.query.vo.DataCenterJoinVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class NetworksRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/networks"; + + @Inject + NetworkDao networkDao; + + @Inject + DataCenterJoinDao dataCenterJoinDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/networks/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = NetworkVOToNetworkConverter.toNetworkList(listNetworks(), this::getZoneById); + final Networks response = new Networks(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listNetworks() { + return networkDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final NetworkVO vo = networkDao.findByUuid(id); + if (vo == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + Network response = NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); + + io.getWriter().write(resp, 200, response, outFormat); + } + + private DataCenterJoinVO getZoneById(Long zoneId) { + if (zoneId == null) { + return null; + } + return dataCenterJoinDao.findById(zoneId); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index dc92f58715f5..02c314c08eb1 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -208,5 +208,4 @@ private HostJoinVO getHostById(Long hostId) { } return hostJoinDao.findById(hostId); } - } \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java new file mode 100644 index 000000000000..9c2ffcca912b --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.api.converter.NetworkVOToVnicProfileConverter; +import org.apache.cloudstack.veeam.api.dto.VnicProfile; +import org.apache.cloudstack.veeam.api.dto.VnicProfiles; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.api.query.dao.DataCenterJoinDao; +import com.cloud.api.query.vo.DataCenterJoinVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; + +public class VnicProfilesRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/vnicprofiles"; + + @Inject + NetworkDao networkDao; + + @Inject + DataCenterJoinDao dataCenterJoinDao; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/vnicprofiles/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = NetworkVOToVnicProfileConverter.toVnicProfileList(listNetworks(), this::getZoneById); + final VnicProfiles response = new VnicProfiles(result); + + io.getWriter().write(resp, 200, response, outFormat); + } + + protected List listNetworks() { + return networkDao.listAll(); + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + final NetworkVO vo = networkDao.findByUuid(id); + if (vo == null) { + io.notFound(resp, "DataCenter not found: " + id, outFormat); + return; + } + VnicProfile response = NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); + + io.getWriter().write(resp, 200, response, outFormat); + } + + private DataCenterJoinVO getZoneById(Long zoneId) { + if (zoneId == null) { + return null; + } + return dataCenterJoinDao.findById(zoneId); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java index 54176d4004ae..3a2c9be5b486 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -19,7 +19,6 @@ import java.util.Collections; import java.util.List; -import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -31,11 +30,12 @@ import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.Ref; +import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenterVO; +import com.cloud.utils.UuidUtils; public class ClusterVOToClusterConverter { - public static Cluster toCluster(final ClusterVO vo, final Function dataCenterResolver) { + public static Cluster toCluster(final ClusterVO vo, final Function dataCenterResolver) { final Cluster c = new Cluster(); final String basePath = VeeamControlService.ContextPath.value(); @@ -131,7 +131,7 @@ public static Cluster toCluster(final ClusterVO vo, final Function pod -> zone) if (dataCenterResolver != null) { - final DataCenterVO zone = dataCenterResolver.apply(vo.getDataCenterId()); + final DataCenterJoinVO zone = dataCenterResolver.apply(vo.getDataCenterId()); if (zone != null) { c.dataCenter = Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + zone.getUuid(), zone.getUuid()); } @@ -157,7 +157,7 @@ public static Cluster toCluster(final ClusterVO vo, final Function toClusterList(final List voList, - final Function dataCenterResolver) { + final Function dataCenterResolver) { return voList.stream() .map(vo -> toCluster(vo, dataCenterResolver)) .collect(Collectors.toList()); @@ -165,6 +165,6 @@ public static List toClusterList(final List voList, private static String stableUuid(final String key) { // deterministic UUID, so the same ClusterVO maps to same "ovirt id" every time - return UUID.nameUUIDFromBytes(key.getBytes()).toString(); + return UuidUtils.nameUUIDFromBytes(key.getBytes()).toString(); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java similarity index 91% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java index c39b91a9684c..465420fc9841 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterVOToDataCenterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java @@ -29,11 +29,11 @@ import org.apache.cloudstack.veeam.api.dto.SupportedVersions; import org.apache.cloudstack.veeam.api.dto.Version; -import com.cloud.dc.DataCenterVO; +import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.org.Grouping; -public class DataCenterVOToDataCenterConverter { - public static DataCenter toDataCenter(final DataCenterVO zone) { +public class DataCenterJoinVOToDataCenterConverter { + public static DataCenter toDataCenter(final DataCenterJoinVO zone) { final String id = zone.getUuid(); final String basePath = VeeamControlService.ContextPath.value(); final String href = basePath + DataCentersRouteHandler.BASE_ROUTE + DataCentersRouteHandler.BASE_ROUTE + "/" + id; @@ -72,9 +72,9 @@ public static DataCenter toDataCenter(final DataCenterVO zone) { return dc; } - public static List toDCList(final List srcList) { + public static List toDCList(final List srcList) { return srcList.stream() - .map(DataCenterVOToDataCenterConverter::toDataCenter) + .map(DataCenterJoinVOToDataCenterConverter::toDataCenter) .collect(Collectors.toList()); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java new file mode 100644 index 000000000000..32c9c3040e91 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ClustersRouteHandler; +import org.apache.cloudstack.veeam.api.HostsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Cpu; +import org.apache.cloudstack.veeam.api.dto.Host; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Topology; + +import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.host.Status; +import com.cloud.resource.ResourceState; + +public class HostJoinVOToHostConverter { + + /** + * Convert CloudStack HostJoinVO -> oVirt-like Host. + * + * @param vo HostJoinVO from listHosts (join query) + */ + public static Host toHost(final HostJoinVO vo) { + final Host h = new Host(); + + final String hostUuid = vo.getUuid(); + + h.setId(hostUuid); + final String basePath = VeeamControlService.ContextPath.value(); + h.setHref(basePath + HostsRouteHandler.BASE_ROUTE + "/" + hostUuid); + + // --- name / address --- + // Prefer DNS name if set; otherwise fall back to IP + final String name = vo.getName() != null ? vo.getName() : ("host-" + hostUuid); + h.setName(name); + + String addr = vo.getPrivateIpAddress(); + h.setAddress(addr); + + h.setStatus(mapStatus(vo)); + h.setExternalStatus("ok"); + + // --- cluster --- + final String clusterUuid = vo.getClusterUuid(); + h.setCluster(Ref.of(basePath + ClustersRouteHandler.BASE_ROUTE + "/" + clusterUuid, clusterUuid)); + + // --- CPU --- + final Cpu cpu = new Cpu(); + + + final Topology topo = new Topology(); + // oVirt topology: sockets/cores/threads. We approximate. + // If CloudStack has cpuNumber = total cores, treat as sockets count w/ 1 core, 1 thread. + topo.sockets = vo.getCpuSockets(); + topo.cores = vo.getCpus(); + topo.threads = 1; + + // --- Memory --- + h.setMemory(String.valueOf(vo.getTotalMemory())); + h.setMaxSchedulingMemory(String.valueOf(vo.getTotalMemory())); + + // --- OS / versions (optional placeholders) --- + // If you want, you can set conservative defaults to match oVirt shape. + h.setType("rhel"); + h.setAutoNumaStatus("unknown"); + h.setKdumpStatus("disabled"); + h.setNumaSupported("false"); + h.setReinstallationRequired("false"); + h.setUpdateAvailable("false"); + + // --- links/actions --- + // Start minimal (empty). Add actions only if Veeam tries to follow them. + h.setActions(null); + h.setLink(Collections.emptyList()); + + return h; + } + + public static List toHostList(final List vos) { + return vos.stream().map(HostJoinVOToHostConverter::toHost).collect(Collectors.toList()); + } + + private static String mapStatus(final HostJoinVO vo) { + // CloudStack examples: + // state: Up/Down/Maintenance/Error/Disconnected + // status: Up/Down/Connecting/etc + if (vo.isInMaintenanceStates()) return "maintenance"; + if (Status.Up.equals(vo.getStatus()) && ResourceState.Enabled.equals(vo.getResourceState())) return "up"; + + // Default + return "down"; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java new file mode 100644 index 000000000000..85775b3d6cfe --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; +import org.apache.cloudstack.veeam.api.NetworksRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Network; +import org.apache.cloudstack.veeam.api.dto.NetworkUsages; +import org.apache.cloudstack.veeam.api.dto.Ref; + +import com.cloud.api.query.vo.DataCenterJoinVO; +import com.cloud.network.dao.NetworkVO; + +public class NetworkVOToNetworkConverter { + public static Network toNetwork(final NetworkVO vo, final Function dcResolver) { + final Network dto = new Network(); + + final String networkUuid = vo.getUuid(); + dto.setId(networkUuid); + final String basePath = VeeamControlService.ContextPath.value(); + dto.setHref(basePath + NetworksRouteHandler.BASE_ROUTE + "/" + networkUuid); + + String name = vo.getName() != null ? vo.getName() : vo.getTrafficType().name() + "-" + networkUuid; + dto.setName(name); + dto.setDescription(vo.getDisplayText()); + dto.setComment(""); + + dto.setMtu(String.valueOf(vo.getPrivateMtu() != null ? vo.getPrivateMtu() : 0)); + dto.setPortIsolation("false"); + dto.setStp("false"); + + dto.setUsages(new NetworkUsages(List.of("vm"))); + + // Best-effort mapping for vdsm_name + dto.setVdsmName(dto.getName()); + + // zone -> oVirt datacenter ref + if (dcResolver != null) { + final DataCenterJoinVO dc = dcResolver.apply(vo.getDataCenterId()); + if (dc != null) { + final String dcUuid = dc.getUuid(); + if (dcUuid != null && !dcUuid.isEmpty()) { + dto.setDataCenter(Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + dcUuid, dcUuid)); + } + } + } + + dto.setLink(Collections.emptyList()); + + return dto; + } + + public static List toNetworkList(final List vos, + final Function dcResolver) { + return vos.stream() + .map(vo -> toNetwork(vo, dcResolver)) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java new file mode 100644 index 000000000000..1dfbb811dc60 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; +import org.apache.cloudstack.veeam.api.NetworksRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.VnicProfile; + +import com.cloud.api.query.vo.DataCenterJoinVO; +import com.cloud.network.dao.NetworkVO; + +public class NetworkVOToVnicProfileConverter { + public static VnicProfile toVnicProfile(final NetworkVO vo, final Function dcResolver) { + final VnicProfile vnicProfile = new VnicProfile(); + + final String networkUuid = vo.getUuid(); + vnicProfile.setId(networkUuid); + final String basePath = VeeamControlService.ContextPath.value(); + vnicProfile.setHref(basePath + NetworksRouteHandler.BASE_ROUTE + "/" + networkUuid); + vnicProfile.setId(networkUuid); + String name = vo.getName() != null ? vo.getName() : vo.getTrafficType().name() + "-" + networkUuid; + vnicProfile.setName(name); + vnicProfile.setNetwork(Ref.of(basePath + NetworksRouteHandler.BASE_ROUTE + "/" + networkUuid, networkUuid)); + vnicProfile.setDescription(vo.getDisplayText()); + + // zone -> oVirt datacenter ref + if (dcResolver != null) { + final DataCenterJoinVO dc = dcResolver.apply(vo.getDataCenterId()); + if (dc != null) { + final String dcUuid = dc.getUuid(); + if (dcUuid != null && !dcUuid.isEmpty()) { + vnicProfile.setDataCenter(Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + dcUuid, dcUuid)); + } + } + } + return vnicProfile; + } + + public static List toVnicProfileList(final List vos, final Function dcResolver) { + return vos.stream() + .map(vo -> toVnicProfile(vo, dcResolver)) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java index 2571f32111f4..7282cc6469b8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java @@ -46,7 +46,7 @@ public final class Api { public SpecialObjects specialObjects; @JacksonXmlProperty(localName = "summary") - public Summary summary; + public ApiSummary summary; // Keep as String to avoid timezone/date parsing friction; you control formatting. @JacksonXmlProperty(localName = "time") diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java similarity index 95% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java index 992590f5f978..ba0618f6a9dc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Summary.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -public final class Summary { +public final class ApiSummary { @JacksonXmlProperty(localName = "hosts") public SummaryCount hosts; @@ -35,5 +35,5 @@ public final class Summary { @JacksonXmlProperty(localName = "vms") public SummaryCount vms; - public Summary() {} + public ApiSummary() {} } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java new file mode 100644 index 000000000000..c95cab88de36 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Certificate { + @JsonProperty("organization") + private String organization; + + @JsonProperty("subject") + private String subject; + + public String getOrganization() { return organization; } + public void setOrganization(String organization) { this.organization = organization; } + public String getSubject() { return subject; } + public void setSubject(String subject) { this.subject = subject; } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java index bc3859d89980..79c6504a9269 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java @@ -18,9 +18,15 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class Cpu { + @JsonProperty("name") + private String name; + + @JsonProperty("speed") + private Integer speed; public String architecture; public Topology topology; @@ -30,4 +36,9 @@ public Cpu(final String architecture, final Topology topology) { this.architecture = architecture; this.topology = topology; } + + public String getName() { return name; } + public void setName(String name) { this.name = name; } + public Integer getSpeed() { return speed; } + public void setSpeed(Integer speed) { this.speed = speed; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java new file mode 100644 index 000000000000..83fb6d8469dd --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class HardwareInformation { + @JsonProperty("manufacturer") + private String manufacturer; + + @JsonProperty("product_name") + private String productName; + + @JsonProperty("serial_number") + private String serialNumber; + + @JsonProperty("uuid") + private String uuid; + + @JsonProperty("version") + private String version; + + public String getManufacturer() { return manufacturer; } + public void setManufacturer(String manufacturer) { this.manufacturer = manufacturer; } + public String getProductName() { return productName; } + public void setProductName(String productName) { this.productName = productName; } + public String getSerialNumber() { return serialNumber; } + public void setSerialNumber(String serialNumber) { this.serialNumber = serialNumber; } + public String getUuid() { return uuid; } + public void setUuid(String uuid) { this.uuid = uuid; } + public String getVersion() { return version; } + public void setVersion(String version) { this.version = version; } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java new file mode 100644 index 000000000000..5a696d0152d7 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Host { + + @JsonProperty("address") + private String address; + + @JsonProperty("auto_numa_status") + private String autoNumaStatus; + + @JsonProperty("certificate") + private Certificate certificate; + + @JsonProperty("cpu") + private Cpu cpu; + + @JsonProperty("external_status") + private String externalStatus; + + @JsonProperty("hardware_information") + private HardwareInformation hardwareInformation; + + @JsonProperty("kdump_status") + private String kdumpStatus; + + @JsonProperty("libvirt_version") + private Version libvirtVersion; + + @JsonProperty("max_scheduling_memory") + private String maxSchedulingMemory; + + @JsonProperty("memory") + private String memory; + + @JsonProperty("numa_supported") + private String numaSupported; + + @JsonProperty("os") + private Os os; + + @JsonProperty("port") + private String port; + + @JsonProperty("protocol") + private String protocol; + + @JsonProperty("reinstallation_required") + private String reinstallationRequired; + + @JsonProperty("status") + private String status; + + @JsonProperty("summary") + private ApiSummary summary; + + @JsonProperty("type") + private String type; + + @JsonProperty("update_available") + private String updateAvailable; + + @JsonProperty("version") + private Version version; + + @JsonProperty("vgpu_placement") + private String vgpuPlacement; + + @JsonProperty("cluster") + private Ref cluster; + + @JsonProperty("actions") + private Actions actions; + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("link") + private List link; + + @JsonProperty("href") + private String href; + + @JsonProperty("id") + private String id; + + // getters/setters (generate via IDE) + public String getAddress() { return address; } + public void setAddress(String address) { this.address = address; } + public String getAutoNumaStatus() { return autoNumaStatus; } + public void setAutoNumaStatus(String autoNumaStatus) { this.autoNumaStatus = autoNumaStatus; } + public Certificate getCertificate() { return certificate; } + public void setCertificate(Certificate certificate) { this.certificate = certificate; } + public Cpu getCpu() { return cpu; } + public void setCpu(Cpu cpu) { this.cpu = cpu; } + public String getExternalStatus() { return externalStatus; } + public void setExternalStatus(String externalStatus) { this.externalStatus = externalStatus; } + public HardwareInformation getHardwareInformation() { return hardwareInformation; } + public void setHardwareInformation(HardwareInformation hardwareInformation) { this.hardwareInformation = hardwareInformation; } + public String getKdumpStatus() { return kdumpStatus; } + public void setKdumpStatus(String kdumpStatus) { this.kdumpStatus = kdumpStatus; } + public Version getLibvirtVersion() { return libvirtVersion; } + public void setLibvirtVersion(Version libvirtVersion) { this.libvirtVersion = libvirtVersion; } + public String getMaxSchedulingMemory() { return maxSchedulingMemory; } + public void setMaxSchedulingMemory(String maxSchedulingMemory) { this.maxSchedulingMemory = maxSchedulingMemory; } + public String getMemory() { return memory; } + public void setMemory(String memory) { this.memory = memory; } + public String getNumaSupported() { return numaSupported; } + public void setNumaSupported(String numaSupported) { this.numaSupported = numaSupported; } + public Os getOs() { return os; } + public void setOs(Os os) { this.os = os; } + public String getPort() { return port; } + public void setPort(String port) { this.port = port; } + public String getProtocol() { return protocol; } + public void setProtocol(String protocol) { this.protocol = protocol; } + public String getReinstallationRequired() { return reinstallationRequired; } + public void setReinstallationRequired(String reinstallationRequired) { this.reinstallationRequired = reinstallationRequired; } + public String getStatus() { return status; } + public void setStatus(String status) { this.status = status; } + public ApiSummary getSummary() { return summary; } + public void setSummary(ApiSummary summary) { this.summary = summary; } + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public String getUpdateAvailable() { return updateAvailable; } + public void setUpdateAvailable(String updateAvailable) { this.updateAvailable = updateAvailable; } + public Version getVersion() { return version; } + public void setVersion(Version version) { this.version = version; } + public String getVgpuPlacement() { return vgpuPlacement; } + public void setVgpuPlacement(String vgpuPlacement) { this.vgpuPlacement = vgpuPlacement; } + public Ref getCluster() { return cluster; } + public void setCluster(Ref cluster) { this.cluster = cluster; } + public Actions getActions() { return actions; } + public void setActions(Actions actions) { this.actions = actions; } + public String getName() { return name; } + public void setName(String name) { this.name = name; } + public String getComment() { return comment; } + public void setComment(String comment) { this.comment = comment; } + public List getLink() { return link; } + public void setLink(List link) { this.link = link; } + public String getHref() { return href; } + public void setHref(String href) { this.href = href; } + public String getId() { return id; } + public void setId(String id) { this.id = id; } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java new file mode 100644 index 000000000000..ada443f27884 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class HostSummary { + @JsonProperty("active") + private String active; + + @JsonProperty("migrating") + private String migrating; + + @JsonProperty("total") + private String total; + + public String getActive() { return active; } + public void setActive(String active) { this.active = active; } + public String getMigrating() { return migrating; } + public void setMigrating(String migrating) { this.migrating = migrating; } + public String getTotal() { return total; } + public void setTotal(String total) { this.total = total; } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java new file mode 100644 index 000000000000..17b3f77de3ef --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Hosts { + @JsonProperty("host") + private List host; + + public Hosts() {} + public Hosts(List host) { this.host = host; } + + public List getHost() { return host; } + public void setHost(List host) { this.host = host; } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java new file mode 100644 index 000000000000..5c259cc8209f --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Network { + private String mtu; // oVirt prints as string + private String portIsolation; // "false" + private String stp; // "false" + private NetworkUsages usages; // { usage: ["vm"] } + private String vdsmName; + + private Ref dataCenter; + + private String name; + private String description; + private String comment; + + @JsonProperty("link") + private List link; + + private String href; + private String id; + + public Network() {} + + // ---- getters / setters ---- + + public String getMtu() { return mtu; } + public void setMtu(final String mtu) { this.mtu = mtu; } + + public String getPortIsolation() { return portIsolation; } + public void setPortIsolation(final String portIsolation) { this.portIsolation = portIsolation; } + + public String getStp() { return stp; } + public void setStp(final String stp) { this.stp = stp; } + + public NetworkUsages getUsages() { return usages; } + public void setUsages(final NetworkUsages usages) { this.usages = usages; } + + public String getVdsmName() { return vdsmName; } + public void setVdsmName(final String vdsmName) { this.vdsmName = vdsmName; } + + public Ref getDataCenter() { return dataCenter; } + public void setDataCenter(final Ref dataCenter) { this.dataCenter = dataCenter; } + + public String getName() { return name; } + public void setName(final String name) { this.name = name; } + + public String getDescription() { return description; } + public void setDescription(final String description) { this.description = description; } + + public String getComment() { return comment; } + public void setComment(final String comment) { this.comment = comment; } + + public List getLink() { return link; } + public void setLink(final List link) { this.link = link; } + + public String getHref() { return href; } + public void setHref(final String href) { this.href = href; } + + public String getId() { return id; } + public void setId(final String id) { this.id = id; } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java new file mode 100644 index 000000000000..da5e1c2aeec5 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class NetworkUsages { + private List usage; + + public NetworkUsages() { + } + + public NetworkUsages(final List usage) { + this.usage = usage; + } + + public List getUsage() { + return usage; + } + + public void setUsage(final List usage) { + this.usage = usage; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java new file mode 100644 index 000000000000..9b96b6e8c2d1 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Networks { + @JsonProperty("network") + private List network; + + public Networks() {} + public Networks(List network) { this.network = network; } + + public List getNetwork() { return network; } + public void setNetwork(List network) { this.network = network; } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java new file mode 100644 index 000000000000..47247f91af5c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class OsVersion { + @JsonProperty("full_version") + private String fullVersion; + + @JsonProperty("major") + private String major; + + @JsonProperty("minor") + private String minor; + + public String getFullVersion() { return fullVersion; } + public void setFullVersion(String fullVersion) { this.fullVersion = fullVersion; } + public String getMajor() { return major; } + public void setMajor(String major) { this.major = major; } + public String getMinor() { return minor; } + public void setMinor(String minor) { this.minor = minor; } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java new file mode 100644 index 000000000000..a550b41090b5 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * oVirt-like vNIC profile element. + * Every vNIC profile MUST reference exactly one network. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VnicProfile { + + private String href; + private String id; + private String name; + private String description; + + private Ref network; + private Ref dataCenter; + + private List link; + + public VnicProfile() { + } + + public String getHref() { + return href; + } + + public void setHref(final String href) { + this.href = href; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public Ref getNetwork() { + return network; + } + + public void setNetwork(final Ref network) { + this.network = network; + } + + public Ref getDataCenter() { + return dataCenter; + } + + public void setDataCenter(final Ref dataCenter) { + this.dataCenter = dataCenter; + } + + public List getLink() { + return link; + } + + public void setLink(final List link) { + this.link = link; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java new file mode 100644 index 000000000000..d528e946bf6e --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Root container for /ovirt-engine/api/vnicprofiles + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VnicProfiles { + + @JsonProperty("vnic_profile") + private List vnicProfile; + + public VnicProfiles() { + } + + public VnicProfiles(final List vnicProfile) { + this.vnicProfile = vnicProfile; + } + + public List getVnicProfile() { + return vnicProfile; + } + + public void setVnicProfile(final List vnicProfile) { + this.vnicProfile = vnicProfile; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java index 5a83299207de..4e32ef577f3b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java @@ -41,7 +41,7 @@ public class BearerOrBasicAuthFilter implements Filter { // Keep these aligned with SsoService (move to ConfigKeys later) public static final List REQUIRED_SCOPES = List.of("ovirt-app-admin", "ovirt-app-portal"); public static final String ISSUER = "veeam-control"; - private static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; + public static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; @Override public void init(FilterConfig filterConfig) {} @Override public void destroy() {} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java index fcd984ffce08..c80668239990 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -37,7 +37,6 @@ public class SsoService extends ManagerBase implements RouteHandler { private static final String BASE_ROUTE = "/sso"; - private static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; // >= 32 chars recommended private static final long DEFAULT_TTL_SECONDS = 3600; // Replace with your real credential validation (CloudStack account, config, etc.) @@ -104,7 +103,8 @@ protected void handleToken(HttpServletRequest req, HttpServletResponse resp, final long ttl = DEFAULT_TTL_SECONDS; final String token; try { - token = JwtUtil.issueHs256Jwt(BearerOrBasicAuthFilter.ISSUER, username, effectiveScope, ttl, HMAC_SECRET); + token = JwtUtil.issueHs256Jwt(BearerOrBasicAuthFilter.ISSUER, username, effectiveScope, ttl, + BearerOrBasicAuthFilter.HMAC_SECRET); } catch (Exception e) { io.getWriter().write(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, Map.of("error", "server_error", "error_description", "Failed to issue token"), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 1ed843cbb46d..e56009aacd49 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -32,9 +32,12 @@ - + + + + From 81c3b5ba0b5ea54ae38184f938249e293f52b10f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 26 Jan 2026 20:59:00 +0530 Subject: [PATCH 006/129] changes Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/veeam/RouteHandler.java | 24 ++++++++++++++++ .../veeam/api/DisksRouteHandler.java | 28 ++++++++++++++----- .../NetworkVOToVnicProfileConverter.java | 3 +- .../veeam/filter/BearerOrBasicAuthFilter.java | 6 ++-- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index 5e0db99d1614..fa7ab174f2b7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.veeam; +import java.io.BufferedReader; import java.io.IOException; import javax.servlet.http.HttpServletRequest; @@ -40,4 +41,27 @@ default String getSanitizedPath(String path) { } return path; } + + static String getRequestData(HttpServletRequest req) { + String contentType = req.getContentType(); + if (contentType == null) { + return null; + } + String mime = contentType.split(";")[0].trim().toLowerCase(); + if (!"application/json".equals(mime) && !"application/x-www-form-urlencoded".equals(mime)) { + return null; + } + try { + StringBuilder data = new StringBuilder(); + String line; + try (BufferedReader reader = req.getReader()) { + while ((line = reader.readLine()) != null) { + data.append(line); + } + } + return data.toString(); + } catch (IOException ignored) { + return null; + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index ad7aed6455ba..708daf059dbe 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -61,16 +61,22 @@ public boolean canHandle(String method, String path) { @Override public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final String method = req.getMethod(); - if (!"GET".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET", outFormat); - return; - } final String sanitizedPath = getSanitizedPath(path); if (sanitizedPath.equals(BASE_ROUTE)) { - handleGet(req, resp, outFormat, io); - return; + if ("GET".equalsIgnoreCase(method)) { + handleGet(req, resp, outFormat, io); + return; + } + if ("POST".equalsIgnoreCase(method)) { + handlePost(req, resp, outFormat, io); + return; + } } + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); if (idAndSubPath != null) { // /api/disks/{id} @@ -90,7 +96,15 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re final List result = VolumeJoinVOToDiskConverter.toDiskList(listDisks()); final Disks response = new Disks(result); - io.getWriter().write(resp, 200, response, outFormat); + io.getWriter().write(resp, 400, response, outFormat); + } + + public void handlePost(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received POST request on /api/disks endpoint, but method: POST is not supported atm. Request-data: {}", data); + + io.getWriter().write(resp, 400, "Unable to process at the moment", outFormat); } protected List listDisks() { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java index 1dfbb811dc60..b9d660f1fa60 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; import org.apache.cloudstack.veeam.api.NetworksRouteHandler; +import org.apache.cloudstack.veeam.api.VnicProfilesRouteHandler; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.VnicProfile; @@ -37,7 +38,7 @@ public static VnicProfile toVnicProfile(final NetworkVO vo, final Function= exp) return false; - if (scope == null || !hasRequiredScopes(scope)) return false; - - return true; + return scope != null && hasRequiredScopes(scope); } private static boolean hasRequiredScopes(String scope) { From f396c5cc747a70335a6287546a9e4d77343a599a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 15 Jan 2026 10:25:07 +0530 Subject: [PATCH 007/129] Basic working version-1 --- .../admin/backup/CreateImageTransferCmd.java | 87 ++++ .../admin/backup/DeleteVmCheckpointCmd.java | 77 +++ .../admin/backup/FinalizeBackupCmd.java | 79 +++ .../backup/FinalizeImageTransferCmd.java | 67 +++ .../admin/backup/ListImageTransfersCmd.java | 79 +++ .../admin/backup/ListVmCheckpointsCmd.java | 69 +++ .../command/admin/backup/StartBackupCmd.java | 65 +++ .../api/response/BackupResponse.java | 36 ++ .../api/response/CheckpointResponse.java | 50 ++ .../api/response/ImageTransferResponse.java | 104 ++++ .../org/apache/cloudstack/backup/Backup.java | 10 + .../cloudstack/backup/ImageTransfer.java | 53 ++ .../backup/IncrementalBackupService.java | 78 +++ .../backup/CreateImageTransferAnswer.java | 65 +++ .../backup/CreateImageTransferCommand.java | 64 +++ .../cloudstack/backup/StartBackupAnswer.java | 57 +++ .../cloudstack/backup/StartBackupCommand.java | 77 +++ .../cloudstack/backup/StopBackupAnswer.java | 30 ++ .../cloudstack/backup/StopBackupCommand.java | 52 ++ .../main/java/com/cloud/vm/VMInstanceVO.java | 22 + .../apache/cloudstack/backup/BackupVO.java | 60 +++ .../cloudstack/backup/ImageTransferVO.java | 243 ++++++++++ .../backup/dao/ImageTransferDao.java | 30 ++ .../backup/dao/ImageTransferDaoImpl.java | 76 +++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42210to42300.sql | 40 ++ ...virtCreateImageTransferCommandWrapper.java | 58 +++ .../LibvirtStartBackupCommandWrapper.java | 159 ++++++ .../LibvirtStopBackupCommandWrapper.java | 69 +++ .../cloudstack/backup/BackupManagerImpl.java | 7 + .../backup/IncrementalBackupServiceImpl.java | 456 ++++++++++++++++++ .../spring-server-core-managers-context.xml | 2 + tools/apidoc/gen_toc.py | 9 + 33 files changed, 2431 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java create mode 100644 api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java create mode 100644 server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java new file mode 100644 index 000000000000..dab2e7459ca6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -0,0 +1,87 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "createImageTransfer", + description = "Create image transfer for a disk in backup", + responseObject = ImageTransferResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "ID of the backup") + private Long backupId; + + @Parameter(name = ApiConstants.VOLUME_ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + required = true, + description = "ID of the disk/volume") + private Long volumeId; + + @Parameter(name = ApiConstants.DIRECTION, + type = CommandType.STRING, + required = true, + description = "Direction of the transfer: upload, download") + private String direction; + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public String getDirection() { + return direction; + } + + @Override + public void execute() { + ImageTransferResponse response = incrementalBackupService.createImageTransfer(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java new file mode 100644 index 000000000000..a05db27de4df --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -0,0 +1,77 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "deleteVmCheckpoint", + description = "Delete a VM checkpoint", + responseObject = SuccessResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = "checkpointid", + type = CommandType.STRING, + required = true, + description = "Checkpoint ID") + private String checkpointId; + + public Long getVmId() { + return vmId; + } + + public String getCheckpointId() { + return checkpointId; + } + + @Override + public void execute() { + boolean result = incrementalBackupService.deleteVmCheckpoint(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java new file mode 100644 index 000000000000..129c570f7acc --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -0,0 +1,79 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "finalizeBackup", + description = "Finalize a VM backup session", + responseObject = SuccessResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class FinalizeBackupCmd extends BaseCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "ID of the backup") + private Long backupId; + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + boolean result = incrementalBackupService.finalizeBackup(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java new file mode 100644 index 000000000000..b8a21a104e37 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -0,0 +1,67 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "finalizeImageTransfer", + description = "Finalize an image transfer", + responseObject = SuccessResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + required = true, + description = "ID of the image transfer") + private Long imageTransferId; + + public Long getImageTransferId() { + return imageTransferId; + } + + @Override + public void execute() { + boolean result = incrementalBackupService.finalizeImageTransfer(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java new file mode 100644 index 000000000000..99d596312d6c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -0,0 +1,79 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "listImageTransfers", + description = "List image transfers for a backup", + responseObject = ImageTransferResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + description = "ID of the Image Transfer") + private Long id; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + public Long getId() { + return id; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + List responses = incrementalBackupService.listImageTransfers(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java new file mode 100644 index 000000000000..737227bf6c7a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; + +@APICommand(name = "listVmCheckpoints", + description = "List checkpoints for a VM", + responseObject = CheckpointResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() { + List responses = incrementalBackupService.listVmCheckpoints(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java new file mode 100644 index 000000000000..ea8995801849 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -0,0 +1,65 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "startBackup", + description = "Start a VM backup session (oVirt-style incremental backup)", + responseObject = BackupResponse.class, + since = "4.22.0", + authorized = {RoleType.Admin}) +public class StartBackupCmd extends BaseCmd implements AdminCmd { + + @Inject + private IncrementalBackupService incrementalBackupService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() { + BackupResponse response = incrementalBackupService.startBackup(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index b855bfe40b8d..f1564843ae36 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -127,6 +127,18 @@ public class BackupResponse extends BaseResponse { @Param(description = "Indicates whether the VM from which the backup was taken is expunged or not", since = "4.22.0") private Boolean isVmExpunged; + @SerializedName("from_checkpoint_id") + @Param(description = "Previous active checkpoint id for incremental backups", since = "4.22.0") + private String fromCheckpointId; + + @SerializedName("to_checkpoint_id") + @Param(description = "Next checkpoint id for incremental backups", since = "4.22.0") + private String toCheckpointId; + + @SerializedName(ApiConstants.HOST_ID) + @Param(description = "Host ID where the backup is running", since = "4.22.0") + private String hostId; + public String getId() { return id; } @@ -314,4 +326,28 @@ public void setVmOfferingRemoved(Boolean vmOfferingRemoved) { public void setVmExpunged(Boolean isVmExpunged) { this.isVmExpunged = isVmExpunged; } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + public String getFromCheckpointId() { + return this.fromCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + public String getToCheckpointId() { + return this.toCheckpointId; + } + + public void setHostId(String hostId) { + this.hostId = hostId; + } + + public String getHostId() { + return this.hostId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java new file mode 100644 index 000000000000..40be9d6d6d0a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -0,0 +1,50 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class CheckpointResponse extends BaseResponse { + + @SerializedName("checkpointid") + @Param(description = "the checkpoint ID") + private String checkpointId; + + @SerializedName("createtime") + @Param(description = "the checkpoint creation time") + private Long createTime; + + @SerializedName("isactive") + @Param(description = "whether this is the active checkpoint") + private Boolean isActive; + + public void setCheckpointId(String checkpointId) { + this.checkpointId = checkpointId; + } + + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java new file mode 100644 index 000000000000..15576e8f1012 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -0,0 +1,104 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.backup.ImageTransfer; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = ImageTransfer.class) +public class ImageTransferResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the image transfer") + private String id; + + @SerializedName("backupid") + @Param(description = "the backup ID") + private String backupId; + + @SerializedName("vmid") + @Param(description = "the VM ID") + private String vmId; + + @SerializedName(ApiConstants.VOLUME_ID) + @Param(description = "the disk/volume ID") + private String diskId; + + @SerializedName("devicename") + @Param(description = "the device name (vda, vdb, etc)") + private String deviceName; + + @SerializedName("transferurl") + @Param(description = "the transfer URL") + private String transferUrl; + + @SerializedName("phase") + @Param(description = "the transfer phase") + private String phase; + + @SerializedName("direction") + @Param(description = "the image transfer direction: upload / download") + private String direction; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date created") + private Date created; + + public void setId(String id) { + this.id = id; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public void setVmId(String vmId) { + this.vmId = vmId; + } + + public void setDiskId(String diskId) { + this.diskId = diskId; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 951af9180e7f..014fc3c483b0 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -30,6 +30,16 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { + String getFromCheckpointId(); + + String getToCheckpointId(); + + Long getCheckpointCreateTime(); + + Long getHostId(); + + Integer getNbdPort(); + enum Status { Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged } diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java new file mode 100644 index 000000000000..4a0cd04ea10b --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ImageTransfer extends ControlledEntity, InternalIdentity { + public enum Direction { + upload, download + } + + public enum Phase { + initializing, transferring, finished, failed + } + + String getUuid(); + + long getBackupId(); + + long getVmId(); + + long getDiskId(); + + String getDeviceName(); + + long getHostId(); + + int getNbdPort(); + + String getTransferUrl(); + + Phase getPhase(); + + Direction getDirection(); + + String getSignedTicketId(); +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java new file mode 100644 index 000000000000..02c079626b4f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -0,0 +1,78 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.List; + +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; + +import com.cloud.utils.component.PluggableService; + +/** + * Service for managing oVirt-style incremental backups using libvirt checkpoints + */ +public interface IncrementalBackupService extends PluggableService { + + /** + * Start a backup session for a VM + * Creates a new checkpoint and starts NBD server for pull-mode backup + */ + BackupResponse startBackup(StartBackupCmd cmd); + + /** + * Finalize a backup session + * Stops NBD server, updates checkpoint tracking, deletes old checkpoints + */ + boolean finalizeBackup(FinalizeBackupCmd cmd); + + /** + * Create an image transfer object for a disk + * Registers NBD endpoint with ImageIO (stubbed for POC) + */ + ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); + + /** + * Finalize an image transfer + * Marks transfer as complete (NBD is closed globally in finalize backup) + */ + boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd); + + /** + * List image transfers for a backup + */ + List listImageTransfers(ListImageTransfersCmd cmd); + + /** + * List checkpoints for a VM + */ + List listVmCheckpoints(ListVmCheckpointsCmd cmd); + + /** + * Delete a VM checkpoint (no-op for normal flow, kept for API parity) + */ + boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd); +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java new file mode 100644 index 000000000000..74dc261893c6 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java @@ -0,0 +1,65 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class CreateImageTransferAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + private String phase; + + public CreateImageTransferAnswer() { + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl, String phase) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + this.phase = phase; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public String getPhase() { + return phase; + } + + public void setPhase(String phase) { + this.phase = phase; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java new file mode 100644 index 000000000000..a4905fe46f75 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -0,0 +1,64 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class CreateImageTransferCommand extends Command { + private Long vmId; + private Long backupId; + private Long diskId; + private String deviceName; + private int nbdPort; + + public CreateImageTransferCommand() { + } + + public CreateImageTransferCommand(Long vmId, Long backupId, Long diskId, String deviceName, int nbdPort) { + this.vmId = vmId; + this.backupId = backupId; + this.diskId = diskId; + this.deviceName = deviceName; + this.nbdPort = nbdPort; + } + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + public Long getDiskId() { + return diskId; + } + + public String getDeviceName() { + return deviceName; + } + + public int getNbdPort() { + return nbdPort; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java new file mode 100644 index 000000000000..056cee41df7a --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -0,0 +1,57 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Answer; + +public class StartBackupAnswer extends Answer { + private Long checkpointCreateTime; + private Map deviceMappings; // volumeId -> device name (vda, vdb, etc.) + + public StartBackupAnswer() { + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, + Long checkpointCreateTime, Map deviceMappings) { + super(cmd, success, details); + this.checkpointCreateTime = checkpointCreateTime; + this.deviceMappings = deviceMappings; + } + + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } + + public Map getDeviceMappings() { + return deviceMappings; + } + + public void setDeviceMappings(Map deviceMappings) { + this.deviceMappings = deviceMappings; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java new file mode 100644 index 000000000000..29fbccafb1f1 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -0,0 +1,77 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class StartBackupCommand extends Command { + private String vmName; + private Long vmId; + private String toCheckpointId; + private String fromCheckpointId; + private int nbdPort; + private Map diskVolumePaths; // volumeId -> path mapping + + public StartBackupCommand() { + } + + public StartBackupCommand(String vmName, Long vmId, String toCheckpointId, String fromCheckpointId, + int nbdPort, Map diskVolumePaths) { + this.vmName = vmName; + this.vmId = vmId; + this.toCheckpointId = toCheckpointId; + this.fromCheckpointId = fromCheckpointId; + this.nbdPort = nbdPort; + this.diskVolumePaths = diskVolumePaths; + } + + public String getVmName() { + return vmName; + } + + public Long getVmId() { + return vmId; + } + + public String getToCheckpointId() { + return toCheckpointId; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public int getNbdPort() { + return nbdPort; + } + + public Map getDiskVolumePaths() { + return diskVolumePaths; + } + + public boolean isIncremental() { + return fromCheckpointId != null && !fromCheckpointId.isEmpty(); + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java new file mode 100644 index 000000000000..ce977f31e005 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java @@ -0,0 +1,30 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StopBackupAnswer extends Answer { + + public StopBackupAnswer() { + } + + public StopBackupAnswer(StopBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java new file mode 100644 index 000000000000..d3055021e9de --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java @@ -0,0 +1,52 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopBackupCommand extends Command { + private String vmName; + private Long vmId; + private Long backupId; + + public StopBackupCommand() { + } + + public StopBackupCommand(String vmName, Long vmId, Long backupId) { + this.vmName = vmName; + this.vmId = vmId; + this.backupId = backupId; + } + + public String getVmName() { + return vmName; + } + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java b/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java index 9d5e1b0ff500..1678caaa525b 100644 --- a/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java @@ -202,6 +202,12 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject getBackupVolumeList() { public void setBackupVolumes(String backupVolumes) { this.backupVolumes = backupVolumes; } + + public String getActiveCheckpointId() { + return activeCheckpointId; + } + + public void setActiveCheckpointId(String activeCheckpointId) { + this.activeCheckpointId = activeCheckpointId; + } + + public Long getActiveCheckpointCreateTime() { + return activeCheckpointCreateTime; + } + + public void setActiveCheckpointCreateTime(Long activeCheckpointCreateTime) { + this.activeCheckpointCreateTime = activeCheckpointCreateTime; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 0f8a10fb7be6..4705cd0159bd 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -103,6 +103,21 @@ public class BackupVO implements Backup { @Column(name = "backup_schedule_id") private Long backupScheduleId; + @Column(name = "from_checkpoint_id") + private String fromCheckpointId; + + @Column(name = "to_checkpoint_id") + private String toCheckpointId; + + @Column(name = "checkpoint_create_time") + private Long checkpointCreateTime; + + @Column(name = "host_id") + private Long hostId; + + @Column(name = "nbd_port") + private Integer nbdPort; + @Transient Map details; @@ -288,4 +303,49 @@ public Long getBackupScheduleId() { public void setBackupScheduleId(Long backupScheduleId) { this.backupScheduleId = backupScheduleId; } + + @Override + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + @Override + public String getToCheckpointId() { + return toCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + @Override + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } + + @Override + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } + + @Override + public Integer getNbdPort() { + return nbdPort; + } + + public void setNbdPort(Integer nbdPort) { + this.nbdPort = nbdPort; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java new file mode 100644 index 000000000000..79953e4cffd8 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -0,0 +1,243 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "image_transfer") +public class ImageTransferVO implements ImageTransfer { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "backup_id") + private long backupId; + + @Column(name = "vm_id") + private long vmId; + + @Column(name = "disk_id") + private long diskId; + + @Column(name = "device_name") + private String deviceName; + + @Column(name = "host_id") + private long hostId; + + @Column(name = "nbd_port") + private int nbdPort; + + @Column(name = "transfer_url") + private String transferUrl; + + @Enumerated(value = EnumType.STRING) + @Column(name = "phase") + private Phase phase; + + @Enumerated(value = EnumType.STRING) + @Column(name = "direction") + private Direction direction; + + @Column(name = "signed_ticket_id") + private String signedTicketId; + + @Column(name = "account_id") + Long accountId; + + @Column(name = "domain_id") + Long domainId; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date updated; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public ImageTransferVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public ImageTransferVO(long backupId, long vmId, long diskId, String deviceName, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId) { + this(); + this.backupId = backupId; + this.vmId = vmId; + this.diskId = diskId; + this.deviceName = deviceName; + this.hostId = hostId; + this.nbdPort = nbdPort; + this.phase = phase; + this.direction = direction; + this.accountId = accountId; + this.domainId = domainId; + this.created = new Date(); + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public long getBackupId() { + return backupId; + } + + public void setBackupId(long backupId) { + this.backupId = backupId; + } + + @Override + public long getVmId() { + return vmId; + } + + public void setVmId(long vmId) { + this.vmId = vmId; + } + + @Override + public long getDiskId() { + return diskId; + } + + public void setDiskId(long diskId) { + this.diskId = diskId; + } + + @Override + public String getDeviceName() { + return deviceName; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + @Override + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + @Override + public int getNbdPort() { + return nbdPort; + } + + public void setNbdPort(int nbdPort) { + this.nbdPort = nbdPort; + } + + @Override + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + @Override + public Phase getPhase() { + return phase; + } + + public void setPhase(Phase phase) { + this.phase = phase; + this.updated = new Date(); + } + + @Override + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + @Override + public String getSignedTicketId() { + return signedTicketId; + } + + public void setSignedTicketId(String signedTicketId) { + this.signedTicketId = signedTicketId; + } + + @Override + public Class getEntityType() { + return ImageTransfer.class; + } + + @Override + public String getName() { + return null; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java new file mode 100644 index 000000000000..e76be261cd8b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -0,0 +1,30 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.ImageTransferVO; + +import com.cloud.utils.db.GenericDao; + +public interface ImageTransferDao extends GenericDao { + List listByBackupId(Long backupId); + List listByVmId(Long vmId); + ImageTransferVO findByUuid(String uuid); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java new file mode 100644 index 000000000000..4c426d870ff8 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -0,0 +1,76 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.cloudstack.backup.ImageTransferVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class ImageTransferDaoImpl extends GenericDaoBase implements ImageTransferDao { + + private SearchBuilder backupIdSearch; + private SearchBuilder vmIdSearch; + private SearchBuilder uuidSearch; + + public ImageTransferDaoImpl() { + } + + @PostConstruct + protected void init() { + backupIdSearch = createSearchBuilder(); + backupIdSearch.and("backupId", backupIdSearch.entity().getBackupId(), SearchCriteria.Op.EQ); + backupIdSearch.done(); + + vmIdSearch = createSearchBuilder(); + vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); + vmIdSearch.done(); + + uuidSearch = createSearchBuilder(); + uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); + uuidSearch.done(); + } + + @Override + public List listByBackupId(Long backupId) { + SearchCriteria sc = backupIdSearch.create(); + sc.setParameters("backupId", backupId); + return listBy(sc); + } + + @Override + public List listByVmId(Long vmId) { + SearchCriteria sc = vmIdSearch.create(); + sc.setParameters("vmId", vmId); + return listBy(sc); + } + + @Override + public ImageTransferVO findByUuid(String uuid) { + SearchCriteria sc = uuidSearch.create(); + sc.setParameters("uuid", uuid); + return findOneBy(sc); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index edc14d9fa0cc..fda874745dfa 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -273,6 +273,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 4cb9eb7cb2c4..e0b0ec48a029 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -117,3 +117,43 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + +-- Add checkpoint tracking fields to backups table for incremental backup support +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for this backup session"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'nbd_port', 'INT DEFAULT NULL COMMENT "NBD server port for backup"'); + +-- Add checkpoint tracking fields to vm_instance table for domain recreation +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Active checkpoint id tracked for incremental backups"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Active checkpoint creation time"'); + +-- Create image_transfer table for per-disk image transfers +CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL COMMENT 'uuid', + `account_id` bigint unsigned NOT NULL COMMENT 'Account ID', + `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', + `backup_id` bigint unsigned NOT NULL COMMENT 'Backup ID', + `vm_id` bigint unsigned NOT NULL COMMENT 'VM ID', + `disk_id` bigint unsigned NOT NULL COMMENT 'Disk/Volume ID', + `device_name` varchar(10) NOT NULL COMMENT 'Device name (vda, vdb, etc)', + `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', + `nbd_port` int NOT NULL COMMENT 'NBD port', + `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', + `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', + `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', + `created` datetime NOT NULL COMMENT 'date created', + `updated` datetime COMMENT 'date updated if not null', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + CONSTRAINT `fk_image_transfer__backup_id` FOREIGN KEY (`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__vm_id` FOREIGN KEY (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__disk_id` FOREIGN KEY (`disk_id`) REFERENCES `volumes`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE, + INDEX `i_image_transfer__backup_id`(`backup_id`), + INDEX `i_image_transfer__vm_id`(`vm_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java new file mode 100644 index 000000000000..b4b39fa2c98e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -0,0 +1,58 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +@ResourceWrapper(handles = CreateImageTransferCommand.class) +public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { + String deviceName = cmd.getDeviceName(); + int nbdPort = cmd.getNbdPort(); + + try { + // POC: ImageIO interaction is stubbed out + // In production, this would: + // 1. Register NBD endpoint nbd://127.0.0.1:{nbdPort}/{deviceName} with ImageIO + // 2. Create transfer object in ImageIO + // 3. Get signed ticket and transfer URL + + // For POC, return stub data + String imageTransferId = "transfer-" + cmd.getDiskId(); + String transferUrl = String.format("nbd://127.0.0.1:%d/%s", nbdPort, deviceName); + String phase = "initializing"; + + return new CreateImageTransferAnswer(cmd, true, "Image transfer created (stub)", + imageTransferId, transferUrl, phase); + + } catch (Exception e) { + return new CreateImageTransferAnswer(cmd, false, "Error creating image transfer: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java new file mode 100644 index 000000000000..ef1d3546f04b --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -0,0 +1,159 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.FileWriter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.StartBackupAnswer; +import org.apache.cloudstack.backup.StartBackupCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartBackupCommand.class) +public class LibvirtStartBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + String toCheckpointId = cmd.getToCheckpointId(); + String fromCheckpointId = cmd.getFromCheckpointId(); + int nbdPort = cmd.getNbdPort(); + + try { + Connect conn = LibvirtConnection.getConnection(); + Domain dm = conn.domainLookupByName(vmName); + + if (dm == null) { + return new StartBackupAnswer(cmd, false, "Domain not found: " + vmName); + } + + DomainInfo info = dm.getInfo(); + if (info.state != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new StartBackupAnswer(cmd, false, "VM is not running"); + } + + // Create backup XML + String backupXml = createBackupXml(cmd, fromCheckpointId, nbdPort); + String checkpointXml = createCheckpointXml(toCheckpointId); + + // Write XMLs to temp files + File backupXmlFile = File.createTempFile("backup-", ".xml"); + File checkpointXmlFile = File.createTempFile("checkpoint-", ".xml"); + + try (FileWriter writer = new FileWriter(backupXmlFile)) { + writer.write(backupXml); + } + try (FileWriter writer = new FileWriter(checkpointXmlFile)) { + writer.write(checkpointXml); + } + + // Execute virsh backup-begin + String backupCmd = String.format("virsh backup-begin %s %s --checkpointxml %s", + vmName, backupXmlFile.getAbsolutePath(), checkpointXmlFile.getAbsolutePath()); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(backupCmd); + String result = script.execute(); + + backupXmlFile.delete(); + checkpointXmlFile.delete(); + + if (result != null) { + return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); + } + + // Get checkpoint creation time - using current time for POC + long checkpointCreateTime = System.currentTimeMillis(); + + // Build device mappings from domblklist + Map deviceMappings = getDeviceMappings(vmName, cmd.getDiskVolumePaths(), resource); + + return new StartBackupAnswer(cmd, true, "Backup started successfully", + checkpointCreateTime, deviceMappings); + + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Error starting backup: " + e.getMessage()); + } + } + + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, int nbdPort) { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + + if (fromCheckpointId != null && !fromCheckpointId.isEmpty()) { + xml.append(" ").append(fromCheckpointId).append("\n"); + } + + xml.append(" \n"); + xml.append(" \n"); + + // Add disk entries - simplified for POC + Map diskPaths = cmd.getDiskVolumePaths(); + int diskIndex = 0; + for (Map.Entry entry : diskPaths.entrySet()) { + String deviceName = "vd" + (char)('a' + diskIndex); + String scratchFile = "/var/tmp/scratch-" + entry.getKey() + ".qcow2"; + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + diskIndex++; + } + + xml.append(" \n"); + xml.append(""); + + return xml.toString(); + } + + private String createCheckpointXml(String checkpointId) { + return "\n" + + " " + checkpointId + "\n" + + ""; + } + + private Map getDeviceMappings(String vmName, Map diskPaths, + LibvirtComputingResource resource) { + Map mappings = new HashMap<>(); + + // Simplified for POC - map volumeIds to device names in order + int diskIndex = 0; + for (Long volumeId : diskPaths.keySet()) { + String deviceName = "vd" + (char)('a' + diskIndex); + mappings.put(volumeId, deviceName); + diskIndex++; + } + + return mappings; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java new file mode 100644 index 000000000000..1185d89bc0b3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopBackupAnswer; +import org.apache.cloudstack.backup.StopBackupCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.libvirt.Connect; +import org.libvirt.Domain; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopBackupCommand.class) +public class LibvirtStopBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StopBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + + try { + Connect conn = LibvirtConnection.getConnection(); + Domain dm = conn.domainLookupByName(vmName); + + if (dm == null) { + return new StopBackupAnswer(cmd, false, "Domain not found: " + vmName); + } + + // Execute virsh domjobabort + String abortCmd = String.format("virsh domjobabort %s", vmName); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(abortCmd); + String result = script.execute(); + + if (result != null && !result.isEmpty()) { + // Job abort may fail if no job is running, which is acceptable + logger.debug("domjobabort result: " + result); + } + + return new StopBackupAnswer(cmd, true, "Backup stopped successfully"); + + } catch (Exception e) { + return new StopBackupAnswer(cmd, false, "Error stopping backup: " + e.getMessage()); + } + } +} diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index db636c7f0f42..7ff345960f8c 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -2430,6 +2430,13 @@ public BackupResponse createBackupResponse(Backup backup, Boolean listVmDetails) response.setVmDetails(vmDetails); } + if (backup.getFromCheckpointId() != null) { + response.setFromCheckpointId(backup.getFromCheckpointId()); + } + if (backup.getToCheckpointId() != null) { + response.setToCheckpointId(backup.getToCheckpointId()); + } + response.setObjectName("backup"); return response; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java new file mode 100644 index 000000000000..cfc36fa76cda --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -0,0 +1,456 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.commons.collections.CollectionUtils; +import org.joda.time.DateTime; +import org.springframework.stereotype.Component; + +import com.cloud.agent.AgentManager; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.VMInstanceDao; + +@Component +public class IncrementalBackupServiceImpl extends ManagerBase implements IncrementalBackupService { + + @Inject + private VMInstanceDao vmInstanceDao; + + @Inject + private BackupDao backupDao; + + @Inject + private ImageTransferDao imageTransferDao; + + @Inject + private VolumeDao volumeDao; + + @Inject + private AgentManager agentManager; + + @Inject + private BackupOfferingDao backupOfferingDao; + + private static final int NBD_PORT_RANGE_START = 10809; + private static final int NBD_PORT_RANGE_END = 10909; + + private boolean isDummyOffering(Long backupOfferingId) { + if (backupOfferingId == null) { + throw new CloudRuntimeException("VM not assigned a backup offering"); + } + BackupOfferingVO offering = backupOfferingDao.findById(backupOfferingId); + if (offering == null) { + throw new CloudRuntimeException("Backup offering not found: " + backupOfferingId); + } + if ("dummy".equalsIgnoreCase(offering.getName())) { + return true; + } + return false; + } + + @Override + public BackupResponse startBackup(StartBackupCmd cmd) { + Long vmId = cmd.getVmId(); + + // Get VM + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + vmId); + } + + if (vm.getState() != State.Running) { + throw new CloudRuntimeException("VM must be running to start backup"); + } + + // Check if backup already in progress + Backup existingBackup = backupDao.findByVmId(vmId); + if (existingBackup != null && existingBackup.getStatus() == Backup.Status.BackingUp) { + throw new CloudRuntimeException("Backup already in progress for VM: " + vmId); + } + + boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); + + // Create backup record + BackupVO backup = new BackupVO(); + backup.setVmId(vmId); + backup.setName(vmId + "-" + DateTime.now()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + // todo: set to Increment if it is incremental backup + backup.setType("FULL"); + backup.setZoneId(vm.getDataCenterId()); + backup.setStatus(Backup.Status.BackingUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setDate(new Date()); + + // Generate checkpoint IDs + String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); + String fromCheckpointId = vm.getActiveCheckpointId(); // null for first full backup + + backup.setToCheckpointId(toCheckpointId); + backup.setFromCheckpointId(fromCheckpointId); + + // Allocate NBD port + int nbdPort = allocateNbdPort(); + backup.setNbdPort(nbdPort); + backup.setHostId(vm.getHostId()); + + // Persist backup record + backup = backupDao.persist(backup); + + // Get disk volume paths + List volumes = volumeDao.findByInstance(vmId); + Map diskVolumePaths = new HashMap<>(); + for (Volume vol : volumes) { + diskVolumePaths.put(vol.getId(), vol.getPath()); + } + + // Send StartBackupCommand to agent + StartBackupCommand startCmd = new StartBackupCommand( + vm.getInstanceName(), + vmId, + toCheckpointId, + fromCheckpointId, + nbdPort, + diskVolumePaths + ); + + try { + StartBackupAnswer answer; + + if (dummyOffering) { + answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis(), diskVolumePaths); + } else { + answer = (StartBackupAnswer) agentManager.send(vm.getHostId(), startCmd); + } + + if (!answer.getResult()) { + backupDao.remove(backup.getId()); + throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); + } + + // Update backup with checkpoint creation time + backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); + backupDao.update(backup.getId(), backup); + + // Return response + BackupResponse response = new BackupResponse(); + response.setId(backup.getUuid()); + response.setVmId(vm.getUuid()); + response.setStatus(backup.getStatus()); + return response; + + } catch (AgentUnavailableException | OperationTimedoutException e) { + backupDao.remove(backup.getId()); + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } + } + + @Override + public boolean finalizeBackup(FinalizeBackupCmd cmd) { + Long vmId = cmd.getVmId(); + Long backupId = cmd.getBackupId(); + + // Get backup + BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup not found: " + backupId); + } + + if (!backup.getVmId().equals(vmId)) { + throw new CloudRuntimeException("Backup does not belong to VM: " + vmId); + } + + // Get VM + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + vmId); + } + + boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); + + List transfers = imageTransferDao.listByBackupId(backupId); + if (CollectionUtils.isNotEmpty(transfers)) { + throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId); + } + + // Send StopBackupCommand to agent + StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); + + try { + StopBackupAnswer answer; + if (dummyOffering) { + answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); + } else { + answer = (StopBackupAnswer) agentManager.send(vm.getHostId(), stopCmd); + } + + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to stop backup: " + answer.getDetails()); + } + + // Update VM checkpoint tracking + String oldCheckpointId = vm.getActiveCheckpointId(); + vm.setActiveCheckpointId(backup.getToCheckpointId()); + vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); + vmInstanceDao.update(vmId, vm); + + // Delete old checkpoint if exists (POC: skip actual libvirt call) + if (oldCheckpointId != null) { + // In production: send command to delete oldCheckpointId via virsh checkpoint-delete + logger.debug("Would delete old checkpoint: " + oldCheckpointId); + } + + // Delete backup session record + backupDao.remove(backup.getId()); + + return true; + + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } + } + + @Override + public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { + Long backupId = cmd.getBackupId(); + Long volumeId = cmd.getVolumeId(); + + BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup not found: " + backupId); + } + + Volume volume = volumeDao.findById(volumeId); + if (volume == null) { + throw new CloudRuntimeException("Volume not found: " + volumeId); + } + + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + backup.getVmId()); + } + boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); + + // Resolve device name (simplified for POC) + List volumes = volumeDao.findByInstance(backup.getVmId()); + String deviceName = resolveDeviceName(volumes, volumeId); + + // Create CreateImageTransferCommand + CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( + backup.getVmId(), + backupId, + volumeId, + deviceName, + backup.getNbdPort() + ); + + try { + CreateImageTransferAnswer answer; + if (dummyOffering) { + answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda", "initializing"); + } else { + answer = (CreateImageTransferAnswer) agentManager.send(backup.getHostId(), transferCmd); + } + + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to create image transfer: " + answer.getDetails()); + } + + // Create ImageTransfer record + ImageTransferVO imageTransfer = new ImageTransferVO( + backupId, + backup.getVmId(), + volumeId, + deviceName, + backup.getHostId(), + backup.getNbdPort(), + ImageTransferVO.Phase.initializing, + ImageTransfer.Direction.valueOf(cmd.getDirection()), + backup.getAccountId(), + backup.getDomainId() + ); + imageTransfer.setTransferUrl(answer.getTransferUrl()); + imageTransfer.setSignedTicketId(answer.getImageTransferId()); + imageTransfer = imageTransferDao.persist(imageTransfer); + + // Return response + ImageTransferResponse response = new ImageTransferResponse(); + response.setId(imageTransfer.getUuid()); + response.setBackupId(backup.getUuid()); + response.setVmId(vm.getUuid()); + response.setDiskId(volume.getUuid()); + response.setDeviceName(deviceName); + response.setTransferUrl(answer.getTransferUrl()); + response.setPhase(ImageTransferVO.Phase.initializing.toString()); + response.setDirection(imageTransfer.getDirection().toString()); + response.setCreated(imageTransfer.getCreated()); + return response; + + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } + } + + @Override + public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { + Long imageTransferId = cmd.getImageTransferId(); + + ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); + if (imageTransfer == null) { + throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); + } + + // Mark as finished (NBD is closed in backup finalize, not here) + imageTransfer.setPhase(ImageTransferVO.Phase.finished); + imageTransferDao.update(imageTransferId, imageTransfer); + imageTransferDao.remove(imageTransferId); + + return true; + } + + @Override + public List listImageTransfers(ListImageTransfersCmd cmd) { + Long id = cmd.getId(); + Long backupId = cmd.getBackupId(); + + List transfers; + if (id != null) { + transfers = List.of(imageTransferDao.findById(id)); + } else if (backupId != null) { + transfers = imageTransferDao.listByBackupId(backupId); + } else { + transfers = imageTransferDao.listAll(); + } + + return transfers.stream().map(this::toImageTransferResponse).collect(Collectors.toList()); + } + + @Override + public List listVmCheckpoints(ListVmCheckpointsCmd cmd) { + Long vmId = cmd.getVmId(); + + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + vmId); + } + + // Return active checkpoint (POC: simplified, no libvirt query) + List responses = new ArrayList<>(); + if (vm.getActiveCheckpointId() != null) { + CheckpointResponse response = new CheckpointResponse(); + response.setCheckpointId(vm.getActiveCheckpointId()); + response.setCreateTime(vm.getActiveCheckpointCreateTime()); + response.setIsActive(true); + responses.add(response); + } + + return responses; + } + + @Override + public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { + // No-op for normal flow as per spec + // Kept for API parity with oVirt + return true; + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList<>(); + cmdList.add(StartBackupCmd.class); + cmdList.add(FinalizeBackupCmd.class); + cmdList.add(CreateImageTransferCmd.class); + cmdList.add(FinalizeImageTransferCmd.class); + cmdList.add(ListImageTransfersCmd.class); + cmdList.add(ListVmCheckpointsCmd.class); + cmdList.add(DeleteVmCheckpointCmd.class); + return cmdList; + } + + // Helper methods + + private int allocateNbdPort() { + // Simplified port allocation for POC + Random random = new Random(); + return NBD_PORT_RANGE_START + random.nextInt(NBD_PORT_RANGE_END - NBD_PORT_RANGE_START); + } + + private String resolveDeviceName(List volumes, Long targetDiskId) { + // Simplified device name resolution for POC + int index = 0; + for (Volume vol : volumes) { + if (Long.valueOf(vol.getId()).equals(targetDiskId)) { + return "vd" + (char)('a' + index); + } + index++; + } + return "vda"; // fallback + } + + private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTransfer) { + ImageTransferResponse response = new ImageTransferResponse(); + response.setId(imageTransfer.getUuid()); + + BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); + VMInstanceVO vm = vmInstanceDao.findById(imageTransfer.getVmId()); + Volume volume = volumeDao.findById(imageTransfer.getDiskId()); + + if (backup != null) response.setBackupId(backup.getUuid()); + if (vm != null) response.setVmId(vm.getUuid()); + if (volume != null) response.setDiskId(volume.getUuid()); + + response.setDeviceName(imageTransfer.getDeviceName()); + response.setTransferUrl(imageTransfer.getTransferUrl()); + response.setPhase(imageTransfer.getPhase().toString()); + response.setCreated(imageTransfer.getCreated()); + + return response; + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 37d32c0f3905..a8c51fdc77e3 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -347,6 +347,8 @@ + + diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 292f52d809bf..9c521caf1f4c 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -223,6 +223,15 @@ 'Management': 'Management', 'Backup' : 'Backup and Recovery', 'Restore' : 'Backup and Recovery', + 'startBackup' : 'Backup and Recovery', + 'finalizeBackup' : 'Backup and Recovery', + 'createImageTransfer' : 'Backup and Recovery', + 'finalizeImageTransfer' : 'Backup and Recovery', + 'listImageTransfers' : 'Backup and Recovery', + 'listVmCheckpoints' : 'Backup and Recovery', + 'deleteVmCheckpoint' : 'Backup and Recovery', + 'ImageTransfer' : 'Backup and Recovery', + 'VmCheckpoint' : 'Backup and Recovery', 'UnmanagedInstance': 'Virtual Machine', 'KubernetesSupportedVersion': 'Kubernetes Service', 'KubernetesCluster': 'Kubernetes Service', From 73df3cbef7f9110bac8b3d33231cc259e27fdba8 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Mon, 19 Jan 2026 11:53:03 +0530 Subject: [PATCH 008/129] Create volume on the given storage pool --- .../command/user/volume/CreateVolumeCmd.java | 11 ++++++++ .../cloud/storage/VolumeApiServiceImpl.java | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 5bcf3a141178..6371a3598abc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -109,6 +110,12 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC description = "The ID of the Instance; to be used with snapshot Id, Instance to which the volume gets attached after creation") private Long virtualMachineId; + @Parameter(name = ApiConstants.STORAGE_ID, + type = CommandType.UUID, + entityType = StoragePoolResponse.class, + description = "Storage pool ID to create the volume in.") + private Long storageId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -153,6 +160,10 @@ private Long getProjectId() { return projectId; } + public Long getStorageId() { + return storageId; + } + public Boolean getDisplayVolume() { return displayVolume; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 17961dbd955f..68af9750317d 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1046,6 +1046,31 @@ public boolean validateVolumeSizeInBytes(long size) { return true; } + private VolumeVO allocateVolumeOnStorage(Long volumeId, Long storageId) { + DataStore destStore = dataStoreMgr.getDataStore(storageId, DataStoreRole.Primary); + VolumeInfo destVolume = volFactory.getVolume(volumeId, destStore); + try { + AsyncCallFuture createVolumeFuture = volService.createVolumeAsync(destVolume, destStore); + VolumeApiResult createVolumeResult = createVolumeFuture.get(); + if (createVolumeResult.isFailed()) { + logger.debug("Failed to create dest volume {}, volume can be removed", destVolume); + destroyVolume(destVolume.getId()); + destVolume.processEvent(ObjectInDataStoreStateMachine.Event.ExpungeRequested); + destVolume.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + _volsDao.remove(destVolume.getId()); + throw new CloudRuntimeException("Creation of a dest volume failed: " + createVolumeResult.getResult()); + } else { + destVolume = volFactory.getVolume(destVolume.getId(), destStore); + destVolume.processEvent(ObjectInDataStoreStateMachine.Event.CreateRequested); + destVolume.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + } + } catch (Exception e) { + logger.debug("Failed to create dest volume {}", destVolume, e); + throw new CloudRuntimeException("Creation of a dest volume failed: volume needs cleanup"); + } + return null; + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", async = true) @@ -1077,6 +1102,8 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { throw new CloudRuntimeException(message.toString()); } } + } else if (cmd.getStorageId() != null) { + allocateVolumeOnStorage(cmd.getEntityId(), cmd.getStorageId()); } return volume; } catch (Exception e) { From 23ecb1f5ce41bdd4ddf5bb1b97de161f64978f01 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 21 Jan 2026 22:23:13 +0530 Subject: [PATCH 009/129] Image server basic working version in SSVM. --- .../backup/CreateImageTransferCommand.java | 32 +- .../backup/FinalizeImageTransferCommand.java | 40 ++ .../cloudstack/backup/StartBackupCommand.java | 8 +- .../cloudstack/backup/ImageTransferVO.java | 6 +- ...virtCreateImageTransferCommandWrapper.java | 7 +- .../LibvirtStartBackupCommandWrapper.java | 2 +- .../backup/IncrementalBackupServiceImpl.java | 65 +- .../resource/NfsSecondaryStorageResource.java | 115 +++ systemvm/debian/opt/cloud/bin/image_server.py | 678 ++++++++++++++++++ 9 files changed, 916 insertions(+), 37 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java create mode 100644 systemvm/debian/opt/cloud/bin/image_server.py diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index a4905fe46f75..f9dfd256c394 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -20,35 +20,21 @@ import com.cloud.agent.api.Command; public class CreateImageTransferCommand extends Command { - private Long vmId; - private Long backupId; - private Long diskId; + private String transferId; + private String hostIpAddress; private String deviceName; private int nbdPort; public CreateImageTransferCommand() { } - public CreateImageTransferCommand(Long vmId, Long backupId, Long diskId, String deviceName, int nbdPort) { - this.vmId = vmId; - this.backupId = backupId; - this.diskId = diskId; + public CreateImageTransferCommand(Long vmId, String transferId, String hostIpAddress, Long backupId, Long diskId, String deviceName, int nbdPort) { + this.transferId = transferId; + this.hostIpAddress = hostIpAddress; this.deviceName = deviceName; this.nbdPort = nbdPort; } - public Long getVmId() { - return vmId; - } - - public Long getBackupId() { - return backupId; - } - - public Long getDiskId() { - return diskId; - } - public String getDeviceName() { return deviceName; } @@ -57,6 +43,14 @@ public int getNbdPort() { return nbdPort; } + public String getHostIpAddress() { + return hostIpAddress; + } + + public String getTransferId() { + return transferId; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java new file mode 100644 index 000000000000..84d9b1ff8186 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -0,0 +1,40 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class FinalizeImageTransferCommand extends Command { + private String transferId; + + public FinalizeImageTransferCommand() { + } + + public FinalizeImageTransferCommand(String transferId) { + this.transferId = transferId; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index 29fbccafb1f1..ac2cc8af70ae 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -28,18 +28,20 @@ public class StartBackupCommand extends Command { private String fromCheckpointId; private int nbdPort; private Map diskVolumePaths; // volumeId -> path mapping + private String hostIpAddress; public StartBackupCommand() { } public StartBackupCommand(String vmName, Long vmId, String toCheckpointId, String fromCheckpointId, - int nbdPort, Map diskVolumePaths) { + int nbdPort, Map diskVolumePaths, String hostIpAddress) { this.vmName = vmName; this.vmId = vmId; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; this.nbdPort = nbdPort; this.diskVolumePaths = diskVolumePaths; + this.hostIpAddress = hostIpAddress; } public String getVmName() { @@ -70,6 +72,10 @@ public boolean isIncremental() { return fromCheckpointId != null && !fromCheckpointId.isEmpty(); } + public String getHostIpAddress() { + return hostIpAddress; + } + @Override public boolean executeInSequence() { return true; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 79953e4cffd8..4efad8d3fd18 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.backup; import java.util.Date; -import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; @@ -94,11 +93,10 @@ public class ImageTransferVO implements ImageTransfer { private Date removed; public ImageTransferVO() { - this.uuid = UUID.randomUUID().toString(); } - public ImageTransferVO(long backupId, long vmId, long diskId, String deviceName, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId) { - this(); + public ImageTransferVO(String uuid, long backupId, long vmId, long diskId, String deviceName, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId) { + this.uuid = uuid; this.backupId = backupId; this.vmId = vmId; this.diskId = diskId; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index b4b39fa2c98e..1c3ec2ae3dc6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -43,13 +43,12 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r // 2. Create transfer object in ImageIO // 3. Get signed ticket and transfer URL - // For POC, return stub data - String imageTransferId = "transfer-" + cmd.getDiskId(); - String transferUrl = String.format("nbd://127.0.0.1:%d/%s", nbdPort, deviceName); + String hostIpAddress = cmd.getHostIpAddress(); + String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, deviceName); String phase = "initializing"; return new CreateImageTransferAnswer(cmd, true, "Image transfer created (stub)", - imageTransferId, transferUrl, phase); + cmd.getTransferId(), transferUrl, phase); } catch (Exception e) { return new CreateImageTransferAnswer(cmd, false, "Error creating image transfer: " + e.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index ef1d3546f04b..57fb39473a2b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -114,7 +114,7 @@ private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, xml.append(" ").append(fromCheckpointId).append("\n"); } - xml.append(" \n"); + xml.append(String.format(" \n", cmd.getHostIpAddress(), nbdPort)); xml.append(" \n"); // Add disk entries - simplified for POC diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index cfc36fa76cda..0723b49bd2e2 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -41,13 +41,18 @@ import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.commons.collections.CollectionUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.host.dao.HostDao; import com.cloud.storage.Volume; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.component.ManagerBase; @@ -77,8 +82,15 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject private BackupOfferingDao backupOfferingDao; + @Inject + private HostDao hostDao; + + @Inject + EndPointSelector _epSelector; + private static final int NBD_PORT_RANGE_START = 10809; private static final int NBD_PORT_RANGE_END = 10909; + private static final boolean DATAPLANE_PROXY_MODE = true; private boolean isDummyOffering(Long backupOfferingId) { if (backupOfferingId == null) { @@ -151,14 +163,15 @@ public BackupResponse startBackup(StartBackupCmd cmd) { diskVolumePaths.put(vol.getId(), vol.getPath()); } - // Send StartBackupCommand to agent + Host host = hostDao.findById(vm.getHostId()); StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), vmId, toCheckpointId, fromCheckpointId, nbdPort, - diskVolumePaths + diskVolumePaths, + host.getPrivateIpAddress() ); try { @@ -282,9 +295,13 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { List volumes = volumeDao.findByInstance(backup.getVmId()); String deviceName = resolveDeviceName(volumes, volumeId); + String transferId = UUID.randomUUID().toString(); + Host host = hostDao.findById(backup.getHostId()); // Create CreateImageTransferCommand CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( backup.getVmId(), + transferId, + host.getPrivateIpAddress(), backupId, volumeId, deviceName, @@ -295,6 +312,9 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { CreateImageTransferAnswer answer; if (dummyOffering) { answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda", "initializing"); + } else if (DATAPLANE_PROXY_MODE) { + EndPoint ssvm = _epSelector.findSsvm(backup.getZoneId()); + answer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); } else { answer = (CreateImageTransferAnswer) agentManager.send(backup.getHostId(), transferCmd); } @@ -305,6 +325,7 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { // Create ImageTransfer record ImageTransferVO imageTransfer = new ImageTransferVO( + transferId, backupId, backup.getVmId(), volumeId, @@ -347,10 +368,32 @@ public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); } - // Mark as finished (NBD is closed in backup finalize, not here) - imageTransfer.setPhase(ImageTransferVO.Phase.finished); - imageTransferDao.update(imageTransferId, imageTransfer); - imageTransferDao.remove(imageTransferId); + BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); + boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); + + FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(imageTransfer.getUuid()); + try { + Answer answer; + if (dummyOffering) { + answer = new Answer(finalizeCmd, true, "Image transfer finalized."); + } else if (DATAPLANE_PROXY_MODE) { + EndPoint ssvm = _epSelector.findSsvm(backup.getZoneId()); + answer = ssvm.sendMessage(finalizeCmd); + } else { + answer = agentManager.send(backup.getHostId(), finalizeCmd); + } + + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to create image transfer: " + answer.getDetails()); + } + + imageTransfer.setPhase(ImageTransferVO.Phase.finished); + imageTransferDao.update(imageTransferId, imageTransfer); + imageTransferDao.remove(imageTransferId); + + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } return true; } @@ -396,8 +439,14 @@ public List listVmCheckpoints(ListVmCheckpointsCmd cmd) { @Override public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { - // No-op for normal flow as per spec - // Kept for API parity with oVirt + // Todo : backend support? + VMInstanceVO vm = vmInstanceDao.findById(cmd.getVmId()); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + cmd.getVmId()); + } + vm.setActiveCheckpointId(null); + vm.setActiveCheckpointCreateTime(null); + vmInstanceDao.update(cmd.getVmId(), vm); return true; } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 8dd2fa23169b..0f8de122d883 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -55,6 +55,10 @@ import javax.naming.ConfigurationException; import com.cloud.agent.api.ConvertSnapshotCommand; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.FinalizeImageTransferCommand; import org.apache.cloudstack.framework.security.keystore.KeystoreManager; import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -338,6 +342,10 @@ public Answer executeRequest(Command cmd) { return execute((ListDataStoreObjectsCommand)cmd); } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { return execute((QuerySnapshotZoneCopyCommand)cmd); + } else if (cmd instanceof CreateImageTransferCommand) { + return execute((CreateImageTransferCommand)cmd); + } else if (cmd instanceof FinalizeImageTransferCommand) { + return execute((FinalizeImageTransferCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -3708,4 +3716,111 @@ protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { return new QuerySnapshotZoneCopyAnswer(cmd, files); } + protected Answer execute(CreateImageTransferCommand cmd) { + if (!_inSystemVM) { + return new CreateImageTransferAnswer(cmd, true, "Not running inside SSVM; skipping image transfer setup."); + } + + final String transferId = cmd.getTransferId(); + + final String hostIp = cmd.getHostIpAddress(); + final String exportName = cmd.getDeviceName(); + final int nbdPort = cmd.getNbdPort(); + + if (StringUtils.isBlank(transferId)) { + return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); + } + if (StringUtils.isBlank(hostIp)) { + return new CreateImageTransferAnswer(cmd, false, "hostIpAddress is empty."); + } + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "deviceName is empty."); + } + if (nbdPort <= 0) { + return new CreateImageTransferAnswer(cmd, false, "Invalid nbdPort: " + nbdPort); + } + + final String imageServerScript = "/opt/cloud/bin/image_server.py"; + final int imageServerPort = 54323; + final String imageServerLogFile = "/var/log/image_server.log"; + + try { + // 1) Write /tmp/ with NBD endpoint details. + final Map payload = new HashMap<>(); + payload.put("host", hostIp); + payload.put("port", nbdPort); + payload.put("export", exportName); + + final String json = new GsonBuilder().create().toJson(payload); + final File transferFile = new File("/tmp", transferId); + FileUtils.writeStringToFile(transferFile, json, "UTF-8"); + + // 2) Start image_server if not already running. + final File scriptFile = new File(imageServerScript); + if (!scriptFile.exists()) { + return new CreateImageTransferAnswer(cmd, false, "Missing image server script: " + imageServerScript); + } + + final Script isRunning = new Script("/bin/bash", logger); + isRunning.add("-c"); + isRunning.add(String.format("pgrep -f '%s.*--port %d' >/dev/null 2>&1", imageServerScript, imageServerPort)); + final String runningResult = isRunning.execute(); + if (runningResult != null) { + try { + ProcessBuilder pb = new ProcessBuilder( + "python3", imageServerScript, + "--listen", "0.0.0.0", + "--port", String.valueOf(imageServerPort) + ); + pb.redirectOutput(ProcessBuilder.Redirect.appendTo(new File(imageServerLogFile))); + pb.redirectErrorStream(true); + pb.start(); + } catch (IOException e) { + logger.warn("Failed to start Image Server"); + return new CreateImageTransferAnswer(cmd, false, "Failed to start image server"); + } + } + final String transferUrl = String.format("http://%s:%d/images/%s", _publicIp, imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on SSVM.", transferId, transferUrl, "initializing"); + } catch (Exception e) { + logger.warn("Failed to prepare image transfer on SSVM", e); + return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on SSVM: " + e.getMessage()); + } + } + + protected Answer execute(FinalizeImageTransferCommand cmd) { + if (!_inSystemVM) { + return new Answer(cmd, true, "Not running inside SSVM; skipping image transfer finalization."); + } + + final String transferId = cmd.getTransferId(); + if (StringUtils.isBlank(transferId)) { + return new Answer(cmd, false, "transferId is empty."); + } + + final File transferFile = new File("/tmp", transferId); + if (transferFile.exists() && !transferFile.delete()) { + return new Answer(cmd, false, "Failed to delete transfer config file: " + transferFile.getAbsolutePath()); + } + + // Stop image_server.py only if /tmp directory is empty. + final File tmpDir = new File("/tmp"); + final File[] tmpEntries = tmpDir.listFiles(); + if (tmpEntries != null && tmpEntries.length == 0) { + final String imageServerScript = "/opt/cloud/bin/image_server.py"; + final int imageServerPort = 54323; + + // Use bash "|| true" so Script returns success even if process isn't running. + final Script stop = new Script("/bin/bash", logger); + stop.add("-c"); + stop.add(String.format("pkill -f '%s.*--port %d' >/dev/null 2>&1 || true", imageServerScript, imageServerPort)); + final String stopResult = stop.execute(); + if (stopResult != null) { + return new Answer(cmd, false, "Failed to stop image server: " + stopResult); + } + } + + return new Answer(cmd, true, "Image transfer finalized."); + } + } diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py new file mode 100644 index 000000000000..2a9013fb4c5f --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python3 +""" +POC "imageio-like" HTTP server backed by NBD over TCP. + +How to run +---------- +- Install dependency: + dnf install python3-libnbd + or + apt install python3-libnbd + +- Run server: + python image_server.py --listen 0.0.0.0 --port 54323 + +Example curl commands +-------------------- +- OPTIONS: + curl -i -X OPTIONS http://127.0.0.1:54323/images/demo + +- GET full image: + curl -v http://127.0.0.1:54323/images/demo -o demo.img + +- GET a byte range: + curl -v -H "Range: bytes=0-1048575" http://127.0.0.1:54323/images/demo -o first_1MiB.bin + +- PUT full image (Content-Length must equal export size exactly): + curl -v -T demo.img http://127.0.0.1:54323/images/demo + +- GET extents (POC-level; may return a single allocated extent): + curl -s http://127.0.0.1:54323/images/demo/extents | jq . + +- POST flush: + curl -s -X POST http://127.0.0.1:54323/images/demo/flush | jq . +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import socket +import threading +import time +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, Optional, Tuple +import nbd + +CHUNK_SIZE = 256 * 1024 # 256 KiB + +# Concurrency limits across ALL images. +MAX_PARALLEL_READS = 8 +MAX_PARALLEL_WRITES = 2 + +_READ_SEM = threading.Semaphore(MAX_PARALLEL_READS) +_WRITE_SEM = threading.Semaphore(MAX_PARALLEL_WRITES) + +# In-memory per-image lock: single lock gates both read and write. +_IMAGE_LOCKS: Dict[str, threading.Lock] = {} +_IMAGE_LOCKS_GUARD = threading.Lock() + + +# Dynamic image_id(transferId) -> NBD export mapping: +# CloudStack writes a JSON file at /tmp/ with: +# {"host": "...", "port": 10809, "export": "vda"} +# +# This server reads that file on-demand. +_CFG_DIR = "/tmp" +_CFG_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {} +_CFG_CACHE_GUARD = threading.Lock() + + +def _json_bytes(obj: Any) -> bytes: + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _get_image_lock(image_id: str) -> threading.Lock: + with _IMAGE_LOCKS_GUARD: + lock = _IMAGE_LOCKS.get(image_id) + if lock is None: + lock = threading.Lock() + _IMAGE_LOCKS[image_id] = lock + return lock + + +def _now_s() -> float: + return time.monotonic() + + +def _safe_transfer_id(image_id: str) -> Optional[str]: + """ + Only allow a single filename component to avoid path traversal. + We intentionally keep validation simple: reject anything containing '/' or '\\'. + """ + if not image_id: + return None + if image_id != os.path.basename(image_id): + return None + if "/" in image_id or "\\" in image_id: + return None + if image_id in (".", ".."): + return None + return image_id + + +def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: + safe_id = _safe_transfer_id(image_id) + if safe_id is None: + return None + + cfg_path = os.path.join(_CFG_DIR, safe_id) + try: + st = os.stat(cfg_path) + except FileNotFoundError: + return None + except OSError as e: + logging.warning("cfg stat failed image_id=%s err=%r", image_id, e) + return None + + with _CFG_CACHE_GUARD: + cached = _CFG_CACHE.get(safe_id) + if cached is not None: + cached_mtime, cached_cfg = cached + # Use cached config if the file hasn't changed. + if float(st.st_mtime) == float(cached_mtime): + return cached_cfg + + try: + with open(cfg_path, "rb") as f: + raw = f.read(4096) + except OSError as e: + logging.warning("cfg read failed image_id=%s err=%r", image_id, e) + return None + + try: + obj = json.loads(raw.decode("utf-8")) + except Exception as e: + logging.warning("cfg parse failed image_id=%s err=%r", image_id, e) + return None + + if not isinstance(obj, dict): + logging.warning("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) + return None + + host = obj.get("host") + port = obj.get("port") + export = obj.get("export") + if not isinstance(host, str) or not host: + logging.warning("cfg missing/invalid host image_id=%s", image_id) + return None + try: + port_i = int(port) + except Exception: + logging.warning("cfg missing/invalid port image_id=%s", image_id) + return None + if port_i <= 0 or port_i > 65535: + logging.warning("cfg out-of-range port image_id=%s port=%r", image_id, port) + return None + if export is not None and (not isinstance(export, str) or not export): + logging.warning("cfg missing/invalid export image_id=%s", image_id) + return None + + cfg: Dict[str, Any] = {"host": host, "port": port_i, "export": export} + + with _CFG_CACHE_GUARD: + _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) + return cfg + + +class _NbdConn: + """ + Small helper to connect to NBD using an already-open TCP socket. + Opens a fresh handle per request, per POC requirements. + """ + + def __init__(self, host: str, port: int, export: Optional[str]): + self._sock = socket.create_connection((host, port)) + self._nbd = nbd.NBD() + + # Select export name if supported/needed. + if export and hasattr(self._nbd, "set_export_name"): + self._nbd.set_export_name(export) + + self._connect_existing_socket(self._sock) + + def _connect_existing_socket(self, sock: socket.socket) -> None: + # Requirement: attach libnbd to an existing socket / FD (no qemu-nbd). + # libnbd python API varies slightly by version, so try common options. + last_err: Optional[BaseException] = None + if hasattr(self._nbd, "connect_socket"): + try: + self._nbd.connect_socket(sock) + return + except Exception as e: # pragma: no cover (depends on binding) + last_err = e + try: + self._nbd.connect_socket(sock.fileno()) + return + except Exception as e2: # pragma: no cover + last_err = e2 + if hasattr(self._nbd, "connect_fd"): + try: + self._nbd.connect_fd(sock.fileno()) + return + except Exception as e: # pragma: no cover + last_err = e + raise RuntimeError( + "Unable to connect libnbd using existing socket/fd; " + f"binding missing connect_socket/connect_fd or call failed: {last_err!r}" + ) + + def size(self) -> int: + return int(self._nbd.get_size()) + + def pread(self, length: int, offset: int) -> bytes: + # Expected signature: pread(length, offset) + try: + return self._nbd.pread(length, offset) + except TypeError: # pragma: no cover (binding differences) + return self._nbd.pread(offset, length) + + def pwrite(self, buf: bytes, offset: int) -> None: + # Expected signature: pwrite(buf, offset) + try: + self._nbd.pwrite(buf, offset) + except TypeError: # pragma: no cover (binding differences) + self._nbd.pwrite(offset, buf) + + def flush(self) -> None: + if hasattr(self._nbd, "flush"): + self._nbd.flush() + return + if hasattr(self._nbd, "fsync"): + self._nbd.fsync() + return + raise RuntimeError("libnbd binding has no flush/fsync method") + + def close(self) -> None: + # Best-effort; bindings may differ. + try: + if hasattr(self._nbd, "shutdown"): + self._nbd.shutdown() + except Exception: + pass + try: + if hasattr(self._nbd, "close"): + self._nbd.close() + except Exception: + pass + try: + self._sock.close() + except Exception: + pass + + def __enter__(self) -> "_NbdConn": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +class Handler(BaseHTTPRequestHandler): + server_version = "imageio-poc/0.1" + + # Keep BaseHTTPRequestHandler from printing noisy default logs + def log_message(self, fmt: str, *args: Any) -> None: + logging.info("%s - - %s", self.address_string(), fmt % args) + + def _send_imageio_headers(self) -> None: + # Include these headers for compatibility with the imageio contract. + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, PUT, OPTIONS") + self.send_header( + "Access-Control-Allow-Headers", + "Range, Content-Range, Content-Type, Content-Length", + ) + self.send_header( + "Access-Control-Expose-Headers", + "Accept-Ranges, Content-Range, Content-Length", + ) + self.send_header("Accept-Ranges", "bytes") + + def _send_json(self, status: int, obj: Any) -> None: + body = _json_bytes(obj) + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + def _send_error_json(self, status: int, message: str) -> None: + self._send_json(status, {"error": message}) + + def _send_range_not_satisfiable(self, size: int) -> None: + # RFC 7233: reply with Content-Range: bytes */ + self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) + self._send_imageio_headers() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Range", f"bytes */{size}") + body = _json_bytes({"error": "range not satisfiable"}) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + def _parse_single_range(self, range_header: str, size: int) -> Tuple[int, int]: + """ + Parse a single HTTP byte range (RFC 7233) and return (start, end_inclusive). + + Supported: + - Range: bytes=START-END + - Range: bytes=START- + - Range: bytes=-SUFFIX + + Raises ValueError for invalid headers. Caller handles 416 vs 400. + """ + if size < 0: + raise ValueError("invalid size") + if not range_header: + raise ValueError("empty Range") + if "," in range_header: + raise ValueError("multiple ranges not supported") + + prefix = "bytes=" + if not range_header.startswith(prefix): + raise ValueError("only bytes ranges supported") + spec = range_header[len(prefix) :].strip() + if "-" not in spec: + raise ValueError("invalid bytes range") + + left, right = spec.split("-", 1) + left = left.strip() + right = right.strip() + + if left == "": + # Suffix range: last N bytes. + if right == "": + raise ValueError("invalid suffix range") + try: + suffix_len = int(right, 10) + except ValueError as e: + raise ValueError("invalid suffix length") from e + if suffix_len <= 0: + raise ValueError("invalid suffix length") + if size == 0: + # Nothing to serve + raise ValueError("unsatisfiable") + if suffix_len >= size: + return 0, size - 1 + return size - suffix_len, size - 1 + + # START is present + try: + start = int(left, 10) + except ValueError as e: + raise ValueError("invalid range start") from e + if start < 0: + raise ValueError("invalid range start") + if start >= size: + raise ValueError("unsatisfiable") + + if right == "": + # START- + return start, size - 1 + + try: + end = int(right, 10) + except ValueError as e: + raise ValueError("invalid range end") from e + if end < start: + raise ValueError("unsatisfiable") + if end >= size: + end = size - 1 + return start, end + + def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: + # Returns (image_id, tail) where tail is: + # None => /images/{id} + # "extents" => /images/{id}/extents + # "flush" => /images/{id}/flush + path = self.path.split("?", 1)[0] + parts = [p for p in path.split("/") if p] + if len(parts) < 2 or parts[0] != "images": + return None, None + image_id = parts[1] + tail = parts[2] if len(parts) >= 3 else None + if len(parts) > 3: + return None, None + return image_id, tail + + def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: + return _load_image_cfg(image_id) + + def do_OPTIONS(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + if self._image_cfg(image_id) is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + self.send_response(HTTPStatus.OK) + self._send_imageio_headers() + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "extents": + self._handle_get_extents(image_id, cfg) + return + if tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + range_header = self.headers.get("Range") + self._handle_get_image(image_id, cfg, range_header) + + def do_PUT(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if self.headers.get("Range") is not None or self.headers.get("Content-Range") is not None: + self._send_error_json( + HTTPStatus.BAD_REQUEST, "Range/Content-Range not supported; full writes only" + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + self._handle_put_image(image_id, cfg, content_length) + + def do_POST(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "flush": + self._handle_post_flush(image_id, cfg) + return + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + + def _handle_get_image( + self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] + ) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _READ_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") + return + + start = _now_s() + bytes_sent = 0 + try: + logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + size = conn.size() + + start_off = 0 + end_off_incl = size - 1 if size > 0 else -1 + status = HTTPStatus.OK + content_length = size + if range_header is not None: + try: + start_off, end_off_incl = self._parse_single_range(range_header, size) + except ValueError as e: + if str(e) == "unsatisfiable": + self._send_range_not_satisfiable(size) + return + if "unsatisfiable" in str(e): + self._send_range_not_satisfiable(size) + return + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") + return + status = HTTPStatus.PARTIAL_CONTENT + content_length = (end_off_incl - start_off) + 1 + + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(content_length)) + if status == HTTPStatus.PARTIAL_CONTENT: + self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") + self.end_headers() + + offset = start_off + end_excl = end_off_incl + 1 + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = conn.pread(to_read, offset) + if not data: + raise RuntimeError("backend returned empty read") + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) + except Exception as e: + # If headers already sent, we can't return JSON reliably; just log. + logging.warning("GET error image_id=%s err=%r", image_id, e) + try: + if not self.wfile.closed: + self.close_connection = True + except Exception: + pass + finally: + _READ_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur + ) + + def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: int) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + bytes_written = 0 + try: + logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + size = conn.size() + if content_length != size: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"Content-Length must equal image size ({size})", + ) + return + + offset = 0 + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {offset} bytes", + ) + return + conn.pwrite(chunk, offset) + offset += len(chunk) + remaining -= len(chunk) + bytes_written += len(chunk) + + # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + except Exception as e: + logging.warning("PUT error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur + ) + + def _handle_get_extents(self, image_id: str, cfg: Dict[str, Any]) -> None: + # Keep deterministic and simple (POC): report entire image allocated. + # No per-image lock required by spec, but we still take it to avoid racing + # with a write and to keep behavior consistent. + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = _now_s() + try: + logging.info("EXTENTS start image_id=%s", image_id) + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + size = conn.size() + self._send_json( + HTTPStatus.OK, + [{"start": 0, "length": size, "allocated": True}], + ) + except Exception as e: + logging.warning("EXTENTS error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = _now_s() - start + logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = _now_s() + try: + logging.info("FLUSH start image_id=%s", image_id) + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + conn.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + except Exception as e: + logging.warning("FLUSH error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = _now_s() - start + logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) + + +def main() -> None: + parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") + parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") + parser.add_argument("--port", type=int, default=54323, help="Port to listen on") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + addr = (args.listen, args.port) + httpd = ThreadingHTTPServer(addr, Handler) + logging.info("listening on http://%s:%d", args.listen, args.port) + logging.info("image configs are read from %s/", _CFG_DIR) + httpd.serve_forever() + + +if __name__ == "__main__": + main() From 5389fe60aa2dc7dfd5748323a86a1ebee0806f31 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 22 Jan 2026 13:28:56 +0530 Subject: [PATCH 010/129] Image server with disk upload --- .../admin/backup/CreateImageTransferCmd.java | 6 +- .../cloudstack/backup/ImageTransfer.java | 6 +- .../backup/CreateImageTransferAnswer.java | 11 +- .../backup/CreateImageTransferCommand.java | 22 +- .../backup/FinalizeImageTransferCommand.java | 14 +- .../cloudstack/backup/StartBackupAnswer.java | 16 +- .../cloudstack/backup/StartBackupCommand.java | 14 +- .../backup/StartNBDServerAnswer.java | 56 ++++ .../backup/StartNBDServerCommand.java | 70 ++++ .../backup/StopNBDServerCommand.java | 52 +++ .../cloudstack/backup/ImageTransferVO.java | 37 +-- .../backup/dao/ImageTransferDao.java | 2 +- .../backup/dao/ImageTransferDaoImpl.java | 24 +- .../META-INF/db/schema-42210to42300.sql | 13 +- ...virtCreateImageTransferCommandWrapper.java | 34 +- .../LibvirtStartBackupCommandWrapper.java | 28 +- .../LibvirtStartNBDServerCommandWrapper.java | 130 ++++++++ .../LibvirtStopNBDServerCommandWrapper.java | 86 +++++ .../backup/IncrementalBackupServiceImpl.java | 300 ++++++++++++------ .../resource/NfsSecondaryStorageResource.java | 154 ++++++--- systemvm/debian/opt/cloud/bin/image_server.py | 4 +- 21 files changed, 797 insertions(+), 282 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index dab2e7459ca6..b67128e47dce 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.context.CallContext; @@ -44,7 +45,6 @@ public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, - required = true, description = "ID of the backup") private Long backupId; @@ -69,8 +69,8 @@ public Long getVolumeId() { return volumeId; } - public String getDirection() { - return direction; + public ImageTransfer.Direction getDirection() { + return ImageTransfer.Direction.valueOf(direction); } @Override diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java index 4a0cd04ea10b..f43be2bdafe4 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -21,6 +21,8 @@ import org.apache.cloudstack.api.InternalIdentity; public interface ImageTransfer extends ControlledEntity, InternalIdentity { + long getDataCenterId(); + public enum Direction { upload, download } @@ -33,12 +35,8 @@ public enum Phase { long getBackupId(); - long getVmId(); - long getDiskId(); - String getDeviceName(); - long getHostId(); int getNbdPort(); diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java index 74dc261893c6..34cf6d4ca34c 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java @@ -22,7 +22,6 @@ public class CreateImageTransferAnswer extends Answer { private String imageTransferId; private String transferUrl; - private String phase; public CreateImageTransferAnswer() { } @@ -32,11 +31,10 @@ public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success } public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details, - String imageTransferId, String transferUrl, String phase) { + String imageTransferId, String transferUrl) { super(cmd, success, details); this.imageTransferId = imageTransferId; this.transferUrl = transferUrl; - this.phase = phase; } public String getImageTransferId() { @@ -55,11 +53,4 @@ public void setTransferUrl(String transferUrl) { this.transferUrl = transferUrl; } - public String getPhase() { - return phase; - } - - public void setPhase(String phase) { - this.phase = phase; - } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index f9dfd256c394..f826c01f3a62 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -22,21 +22,25 @@ public class CreateImageTransferCommand extends Command { private String transferId; private String hostIpAddress; - private String deviceName; + private String exportName; + private String volumePath; private int nbdPort; + private String direction; public CreateImageTransferCommand() { } - public CreateImageTransferCommand(Long vmId, String transferId, String hostIpAddress, Long backupId, Long diskId, String deviceName, int nbdPort) { + public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, String volumePath, int nbdPort, String direction) { this.transferId = transferId; this.hostIpAddress = hostIpAddress; - this.deviceName = deviceName; + this.exportName = exportName; + this.volumePath = volumePath; this.nbdPort = nbdPort; + this.direction = direction; } - public String getDeviceName() { - return deviceName; + public String getExportName() { + return exportName; } public int getNbdPort() { @@ -55,4 +59,12 @@ public String getTransferId() { public boolean executeInSequence() { return true; } + + public String getVolumePath() { + return volumePath; + } + + public String getDirection() { + return direction; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java index 84d9b1ff8186..f1a0285ef6ed 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -21,18 +21,30 @@ public class FinalizeImageTransferCommand extends Command { private String transferId; + private String direction; + private int nbdPort; public FinalizeImageTransferCommand() { } - public FinalizeImageTransferCommand(String transferId) { + public FinalizeImageTransferCommand(String transferId, String direction, int nbdPort) { this.transferId = transferId; + this.direction = direction; + this.nbdPort = nbdPort; } public String getTransferId() { return transferId; } + public int getNbdPort() { + return nbdPort; + } + + public String getDirection() { + return direction; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java index 056cee41df7a..7628fe19698f 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -17,13 +17,11 @@ package org.apache.cloudstack.backup; -import java.util.Map; - import com.cloud.agent.api.Answer; public class StartBackupAnswer extends Answer { private Long checkpointCreateTime; - private Map deviceMappings; // volumeId -> device name (vda, vdb, etc.) + private Boolean isIncremental; public StartBackupAnswer() { } @@ -32,11 +30,9 @@ public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details super(cmd, success, details); } - public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, - Long checkpointCreateTime, Map deviceMappings) { + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, Long checkpointCreateTime) { super(cmd, success, details); this.checkpointCreateTime = checkpointCreateTime; - this.deviceMappings = deviceMappings; } public Long getCheckpointCreateTime() { @@ -47,11 +43,11 @@ public void setCheckpointCreateTime(Long checkpointCreateTime) { this.checkpointCreateTime = checkpointCreateTime; } - public Map getDeviceMappings() { - return deviceMappings; + public Boolean getIncremental() { + return isIncremental; } - public void setDeviceMappings(Map deviceMappings) { - this.deviceMappings = deviceMappings; + public void setIncremental(Boolean incremental) { + isIncremental = incremental; } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index ac2cc8af70ae..ba4daddc116d 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -23,20 +23,18 @@ public class StartBackupCommand extends Command { private String vmName; - private Long vmId; private String toCheckpointId; private String fromCheckpointId; private int nbdPort; - private Map diskVolumePaths; // volumeId -> path mapping + private Map diskVolumePaths; // volumeId -> path mapping private String hostIpAddress; public StartBackupCommand() { } - public StartBackupCommand(String vmName, Long vmId, String toCheckpointId, String fromCheckpointId, - int nbdPort, Map diskVolumePaths, String hostIpAddress) { + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, + int nbdPort, Map diskVolumePaths, String hostIpAddress) { this.vmName = vmName; - this.vmId = vmId; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; this.nbdPort = nbdPort; @@ -48,10 +46,6 @@ public String getVmName() { return vmName; } - public Long getVmId() { - return vmId; - } - public String getToCheckpointId() { return toCheckpointId; } @@ -64,7 +58,7 @@ public int getNbdPort() { return nbdPort; } - public Map getDiskVolumePaths() { + public Map getDiskVolumePaths() { return diskVolumePaths; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java new file mode 100644 index 000000000000..d8c78d3c8807 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartNBDServerAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public StartNBDServerAnswer() { + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java new file mode 100644 index 000000000000..887937ffb4c8 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -0,0 +1,70 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StartNBDServerCommand extends Command { + private String transferId; + private String hostIpAddress; + private String exportName; + private String volumePath; + private int nbdPort; + private String direction; + + public StartNBDServerCommand() { + } + + public StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, int nbdPort, String direction) { + this.transferId = transferId; + this.hostIpAddress = hostIpAddress; + this.exportName = exportName; + this.volumePath = volumePath; + this.nbdPort = nbdPort; + this.direction = direction; + } + + public String getExportName() { + return exportName; + } + + public int getNbdPort() { + return nbdPort; + } + + public String getHostIpAddress() { + return hostIpAddress; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getVolumePath() { + return volumePath; + } + + public String getDirection() { + return direction; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java new file mode 100644 index 000000000000..4f2b6401480e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java @@ -0,0 +1,52 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopNBDServerCommand extends Command { + private String transferId; + private String direction; + private int nbdPort; + + public StopNBDServerCommand() { + } + + public StopNBDServerCommand(String transferId, String direction, int nbdPort) { + this.transferId = transferId; + this.direction = direction; + this.nbdPort = nbdPort; + } + + public String getTransferId() { + return transferId; + } + + public int getNbdPort() { + return nbdPort; + } + + public String getDirection() { + return direction; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 4efad8d3fd18..d9bd1f4c9ba2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -45,15 +45,9 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "backup_id") private long backupId; - @Column(name = "vm_id") - private long vmId; - @Column(name = "disk_id") private long diskId; - @Column(name = "device_name") - private String deviceName; - @Column(name = "host_id") private long hostId; @@ -80,6 +74,9 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "domain_id") Long domainId; + @Column(name = "data_center_id") + Long dataCenterId; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created; @@ -95,18 +92,17 @@ public class ImageTransferVO implements ImageTransfer { public ImageTransferVO() { } - public ImageTransferVO(String uuid, long backupId, long vmId, long diskId, String deviceName, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId) { + public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { this.uuid = uuid; this.backupId = backupId; - this.vmId = vmId; this.diskId = diskId; - this.deviceName = deviceName; this.hostId = hostId; this.nbdPort = nbdPort; this.phase = phase; this.direction = direction; this.accountId = accountId; this.domainId = domainId; + this.dataCenterId = dataCenterId; this.created = new Date(); } @@ -129,15 +125,6 @@ public void setBackupId(long backupId) { this.backupId = backupId; } - @Override - public long getVmId() { - return vmId; - } - - public void setVmId(long vmId) { - this.vmId = vmId; - } - @Override public long getDiskId() { return diskId; @@ -147,15 +134,6 @@ public void setDiskId(long diskId) { this.diskId = diskId; } - @Override - public String getDeviceName() { - return deviceName; - } - - public void setDeviceName(String deviceName) { - this.deviceName = deviceName; - } - @Override public long getHostId() { return hostId; @@ -231,6 +209,11 @@ public long getAccountId() { return accountId; } + @Override + public long getDataCenterId() { + return dataCenterId; + } + public Date getCreated() { return created; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index e76be261cd8b..e5e57c4acdac 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -25,6 +25,6 @@ public interface ImageTransferDao extends GenericDao { List listByBackupId(Long backupId); - List listByVmId(Long vmId); ImageTransferVO findByUuid(String uuid); + ImageTransferVO findByNbdPort(int port); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 4c426d870ff8..57587858661c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -32,8 +32,8 @@ public class ImageTransferDaoImpl extends GenericDaoBase implements ImageTransferDao { private SearchBuilder backupIdSearch; - private SearchBuilder vmIdSearch; private SearchBuilder uuidSearch; + private SearchBuilder nbdPortSearch; public ImageTransferDaoImpl() { } @@ -44,13 +44,13 @@ protected void init() { backupIdSearch.and("backupId", backupIdSearch.entity().getBackupId(), SearchCriteria.Op.EQ); backupIdSearch.done(); - vmIdSearch = createSearchBuilder(); - vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); - vmIdSearch.done(); - uuidSearch = createSearchBuilder(); uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); uuidSearch.done(); + + nbdPortSearch = createSearchBuilder(); + nbdPortSearch.and("nbdPort", nbdPortSearch.entity().getNbdPort(), SearchCriteria.Op.EQ); + nbdPortSearch.done(); } @Override @@ -60,17 +60,17 @@ public List listByBackupId(Long backupId) { return listBy(sc); } - @Override - public List listByVmId(Long vmId) { - SearchCriteria sc = vmIdSearch.create(); - sc.setParameters("vmId", vmId); - return listBy(sc); - } - @Override public ImageTransferVO findByUuid(String uuid) { SearchCriteria sc = uuidSearch.create(); sc.setParameters("uuid", uuid); return findOneBy(sc); } + + @Override + public ImageTransferVO findByNbdPort(int port) { + SearchCriteria sc = nbdPortSearch.create(); + sc.setParameters("nbdPort", port); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index e0b0ec48a029..d3ee808cbacc 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -131,14 +131,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_cre -- Create image_transfer table for per-disk image transfers CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `uuid` varchar(40) NOT NULL COMMENT 'uuid', + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL COMMENT 'uuid', `account_id` bigint unsigned NOT NULL COMMENT 'Account ID', `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', - `backup_id` bigint unsigned NOT NULL COMMENT 'Backup ID', - `vm_id` bigint unsigned NOT NULL COMMENT 'VM ID', + `data_center_id` bigint unsigned NOT NULL COMMENT 'Data Center ID', + `backup_id` bigint unsigned COMMENT 'Backup ID', `disk_id` bigint unsigned NOT NULL COMMENT 'Disk/Volume ID', - `device_name` varchar(10) NOT NULL COMMENT 'Device name (vda, vdb, etc)', `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', `nbd_port` int NOT NULL COMMENT 'NBD port', `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', @@ -151,9 +150,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( PRIMARY KEY (`id`), UNIQUE KEY `uuid` (`uuid`), CONSTRAINT `fk_image_transfer__backup_id` FOREIGN KEY (`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE, - CONSTRAINT `fk_image_transfer__vm_id` FOREIGN KEY (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_image_transfer__disk_id` FOREIGN KEY (`disk_id`) REFERENCES `volumes`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_image_transfer__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE, - INDEX `i_image_transfer__backup_id`(`backup_id`), - INDEX `i_image_transfer__vm_id`(`vm_id`) + INDEX `i_image_transfer__backup_id`(`backup_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 1c3ec2ae3dc6..1db594d169fb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -19,8 +19,8 @@ import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -31,27 +31,31 @@ public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); - @Override - public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { - String deviceName = cmd.getDeviceName(); - int nbdPort = cmd.getNbdPort(); + private CreateImageTransferAnswer handleUpload(CreateImageTransferCommand cmd) { + return new CreateImageTransferAnswer(cmd, false, "Image Upload is not handled by KVM agent"); + } + private CreateImageTransferAnswer handleDownload(CreateImageTransferCommand cmd) { + String exportName = cmd.getExportName(); + int nbdPort = cmd.getNbdPort(); try { - // POC: ImageIO interaction is stubbed out - // In production, this would: - // 1. Register NBD endpoint nbd://127.0.0.1:{nbdPort}/{deviceName} with ImageIO - // 2. Create transfer object in ImageIO - // 3. Get signed ticket and transfer URL - String hostIpAddress = cmd.getHostIpAddress(); - String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, deviceName); - String phase = "initializing"; + String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, exportName); - return new CreateImageTransferAnswer(cmd, true, "Image transfer created (stub)", - cmd.getTransferId(), transferUrl, phase); + return new CreateImageTransferAnswer(cmd, true, "Image transfer created for download", + cmd.getTransferId(), transferUrl); } catch (Exception e) { return new CreateImageTransferAnswer(cmd, false, "Error creating image transfer: " + e.getMessage()); } } + + @Override + public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { + if (cmd.getDirection().equals("download")) { + return handleDownload(cmd); + } else { + return handleUpload(cmd); + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 57fb39473a2b..5013e4d79726 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.FileWriter; -import java.util.HashMap; import java.util.Map; import org.apache.cloudstack.backup.StartBackupAnswer; @@ -95,11 +94,7 @@ public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) // Get checkpoint creation time - using current time for POC long checkpointCreateTime = System.currentTimeMillis(); - // Build device mappings from domblklist - Map deviceMappings = getDeviceMappings(vmName, cmd.getDiskVolumePaths(), resource); - - return new StartBackupAnswer(cmd, true, "Backup started successfully", - checkpointCreateTime, deviceMappings); + return new StartBackupAnswer(cmd, true, "Backup started successfully", checkpointCreateTime); } catch (Exception e) { return new StartBackupAnswer(cmd, false, "Error starting backup: " + e.getMessage()); @@ -118,13 +113,13 @@ private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, xml.append(" \n"); // Add disk entries - simplified for POC - Map diskPaths = cmd.getDiskVolumePaths(); + Map diskPaths = cmd.getDiskVolumePaths(); int diskIndex = 0; - for (Map.Entry entry : diskPaths.entrySet()) { + for (Map.Entry entry : diskPaths.entrySet()) { String deviceName = "vd" + (char)('a' + diskIndex); String scratchFile = "/var/tmp/scratch-" + entry.getKey() + ".qcow2"; xml.append(" \n"); + .append(entry.getKey()).append("\">\n"); xml.append(" \n"); xml.append(" \n"); diskIndex++; @@ -141,19 +136,4 @@ private String createCheckpointXml(String checkpointId) { " " + checkpointId + "\n" + ""; } - - private Map getDeviceMappings(String vmName, Map diskPaths, - LibvirtComputingResource resource) { - Map mappings = new HashMap<>(); - - // Simplified for POC - map volumeIds to device names in order - int diskIndex = 0; - for (Long volumeId : diskPaths.keySet()) { - String deviceName = "vd" + (char)('a' + diskIndex); - mappings.put(volumeId, deviceName); - diskIndex++; - } - - return mappings; - } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java new file mode 100644 index 000000000000..c7f2e8d6d08c --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -0,0 +1,130 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StartNBDServerAnswer; +import org.apache.cloudstack.backup.StartNBDServerCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartNBDServerCommand.class) +public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private StartNBDServerAnswer handleUpload(StartNBDServerCommand cmd) { + String volumePath = cmd.getVolumePath(); + int nbdPort = cmd.getNbdPort(); + String hostIpAddress = cmd.getHostIpAddress(); + String exportName = cmd.getExportName(); + String transferId = cmd.getTransferId(); + + if (volumePath == null || volumePath.isEmpty()) { + return new StartNBDServerAnswer(cmd, false, "Volume path is required for upload"); + } + if (exportName == null || exportName.isEmpty()) { + return new StartNBDServerAnswer(cmd, false, "Export name is required for upload"); + } + if (hostIpAddress == null || hostIpAddress.isEmpty()) { + return new StartNBDServerAnswer(cmd, false, "Host IP address is required for upload"); + } + + String unitName = String.format("qemu-nbd-%d", nbdPort); + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return new StartNBDServerAnswer(cmd, false, "A qemu-nbd service is already running on the port."); + } + + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --bind %s --port %d --persistent %s", + unitName, exportName, hostIpAddress, nbdPort, volumePath + ); + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start qemu-nbd service: %s", startResult)); + return new StartNBDServerAnswer(cmd, false, "Failed to start qemu-nbd service: " + startResult); + } + + // Wait with timeout until the service is up + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("qemu-nbd service %s is now active (attempt %d)", unitName, attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new StartNBDServerAnswer(cmd, false, "Interrupted while waiting for qemu-nbd service to start"); + } + } + + if (!serviceActive) { + logger.error(String.format("qemu-nbd service %s failed to become active within %d seconds", unitName, maxWaitSeconds)); + return new StartNBDServerAnswer(cmd, false, + String.format("qemu-nbd service failed to start within %d seconds", maxWaitSeconds)); + } + + String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, exportName); + return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for upload", + transferId, transferUrl); + } + + private StartNBDServerAnswer handleDownload(StartNBDServerCommand cmd) { + String exportName = cmd.getExportName(); + int nbdPort = cmd.getNbdPort(); + String hostIpAddress = cmd.getHostIpAddress(); + String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, exportName); + + return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for download", + cmd.getTransferId(), transferUrl); + } + + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { + if (cmd.getDirection().equals("download")) { + return handleDownload(cmd); + } else { + return handleUpload(cmd); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java new file mode 100644 index 000000000000..96ac0e7accc7 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopNBDServerCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopNBDServerCommand.class) +public class LibvirtStopNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private Answer handleUpload(StopNBDServerCommand cmd) { + try { + int nbdPort = cmd.getNbdPort(); + String unitName = String.format("qemu-nbd-%d", nbdPort); + + // Check if the service is running + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + // Service is not running, but still reset-failed to clear any stale state + logger.info(String.format("qemu-nbd service %s is not running, resetting failed state", unitName)); + resetService(unitName); + return new Answer(cmd, true, "Image transfer finalized"); + } + + // Stop the systemd service + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + + return new Answer(cmd, true, "Image transfer finalized"); + + } catch (Exception e) { + logger.error("Error finalizing image transfer for upload", e); + return new Answer(cmd, false, "Error finalizing image transfer: " + e.getMessage()); + } + } + + private Answer handleDownload(StopNBDServerCommand cmd) { + return new Answer(cmd, true, "Image transfer finalized"); + } + + @Override + public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { + if (cmd.getDirection().equals("download")) { + return handleDownload(cmd); + } else { + return handleUpload(cmd); + } + + } +} diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 0723b49bd2e2..2eace1ff1ba9 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -43,6 +43,8 @@ import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; @@ -52,8 +54,11 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; +import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.storage.ScopeType; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -85,6 +90,9 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject private HostDao hostDao; + @Inject + private PrimaryDataStoreDao primaryDataStoreDao; + @Inject EndPointSelector _epSelector; @@ -110,7 +118,6 @@ private boolean isDummyOffering(Long backupOfferingId) { public BackupResponse startBackup(StartBackupCmd cmd) { Long vmId = cmd.getVmId(); - // Get VM VMInstanceVO vm = vmInstanceDao.findById(vmId); if (vm == null) { throw new CloudRuntimeException("VM not found: " + vmId); @@ -120,7 +127,6 @@ public BackupResponse startBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("VM must be running to start backup"); } - // Check if backup already in progress Backup existingBackup = backupDao.findByVmId(vmId); if (existingBackup != null && existingBackup.getStatus() == Backup.Status.BackingUp) { throw new CloudRuntimeException("Backup already in progress for VM: " + vmId); @@ -128,45 +134,39 @@ public BackupResponse startBackup(StartBackupCmd cmd) { boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); - // Create backup record BackupVO backup = new BackupVO(); backup.setVmId(vmId); backup.setName(vmId + "-" + DateTime.now()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); - // todo: set to Increment if it is incremental backup - backup.setType("FULL"); backup.setZoneId(vm.getDataCenterId()); backup.setStatus(Backup.Status.BackingUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setDate(new Date()); - // Generate checkpoint IDs String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); - String fromCheckpointId = vm.getActiveCheckpointId(); // null for first full backup + String fromCheckpointId = vm.getActiveCheckpointId(); backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); - // Allocate NBD port int nbdPort = allocateNbdPort(); backup.setNbdPort(nbdPort); backup.setHostId(vm.getHostId()); + // Will be changed later if incremental was done + backup.setType("FULL"); - // Persist backup record backup = backupDao.persist(backup); - // Get disk volume paths - List volumes = volumeDao.findByInstance(vmId); - Map diskVolumePaths = new HashMap<>(); + List volumes = volumeDao.findByInstance(vmId); + Map diskVolumePaths = new HashMap<>(); for (Volume vol : volumes) { - diskVolumePaths.put(vol.getId(), vol.getPath()); + diskVolumePaths.put(vol.getUuid(), vol.getPath()); } Host host = hostDao.findById(vm.getHostId()); StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), - vmId, toCheckpointId, fromCheckpointId, nbdPort, @@ -178,7 +178,7 @@ public BackupResponse startBackup(StartBackupCmd cmd) { StartBackupAnswer answer; if (dummyOffering) { - answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis(), diskVolumePaths); + answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis()); } else { answer = (StartBackupAnswer) agentManager.send(vm.getHostId(), startCmd); } @@ -190,9 +190,12 @@ public BackupResponse startBackup(StartBackupCmd cmd) { // Update backup with checkpoint creation time backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); + if (Boolean.TRUE.equals(answer.getIncremental())) { + // todo: set it in the backend + backup.setType("Incremental"); + } backupDao.update(backup.getId(), backup); - // Return response BackupResponse response = new BackupResponse(); response.setId(backup.getUuid()); response.setVmId(vm.getUuid()); @@ -270,48 +273,35 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } } - @Override - public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { + private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd) { Long backupId = cmd.getBackupId(); Long volumeId = cmd.getVolumeId(); - BackupVO backup = backupDao.findById(backupId); if (backup == null) { throw new CloudRuntimeException("Backup not found: " + backupId); } + boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); Volume volume = volumeDao.findById(volumeId); if (volume == null) { throw new CloudRuntimeException("Volume not found: " + volumeId); } - VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); - if (vm == null) { - throw new CloudRuntimeException("VM not found: " + backup.getVmId()); - } - boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); - - // Resolve device name (simplified for POC) - List volumes = volumeDao.findByInstance(backup.getVmId()); - String deviceName = resolveDeviceName(volumes, volumeId); - String transferId = UUID.randomUUID().toString(); Host host = hostDao.findById(backup.getHostId()); - // Create CreateImageTransferCommand CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( - backup.getVmId(), - transferId, - host.getPrivateIpAddress(), - backupId, - volumeId, - deviceName, - backup.getNbdPort() + transferId, + host.getPrivateIpAddress(), + volume.getUuid(), + null, + backup.getNbdPort(), + cmd.getDirection().toString() ); try { CreateImageTransferAnswer answer; if (dummyOffering) { - answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda", "initializing"); + answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda"); } else if (DATAPLANE_PROXY_MODE) { EndPoint ssvm = _epSelector.findSsvm(backup.getZoneId()); answer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); @@ -323,55 +313,131 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { throw new CloudRuntimeException("Failed to create image transfer: " + answer.getDetails()); } - // Create ImageTransfer record ImageTransferVO imageTransfer = new ImageTransferVO( - transferId, - backupId, - backup.getVmId(), - volumeId, - deviceName, - backup.getHostId(), - backup.getNbdPort(), - ImageTransferVO.Phase.initializing, - ImageTransfer.Direction.valueOf(cmd.getDirection()), - backup.getAccountId(), - backup.getDomainId() + transferId, + backupId, + volumeId, + backup.getHostId(), + backup.getNbdPort(), + ImageTransferVO.Phase.transferring, + ImageTransfer.Direction.download, + backup.getAccountId(), + backup.getDomainId(), + backup.getZoneId() ); imageTransfer.setTransferUrl(answer.getTransferUrl()); imageTransfer.setSignedTicketId(answer.getImageTransferId()); imageTransfer = imageTransferDao.persist(imageTransfer); + return imageTransfer; - // Return response - ImageTransferResponse response = new ImageTransferResponse(); - response.setId(imageTransfer.getUuid()); - response.setBackupId(backup.getUuid()); - response.setVmId(vm.getUuid()); - response.setDiskId(volume.getUuid()); - response.setDeviceName(deviceName); - response.setTransferUrl(answer.getTransferUrl()); - response.setPhase(ImageTransferVO.Phase.initializing.toString()); - response.setDirection(imageTransfer.getDirection().toString()); - response.setCreated(imageTransfer.getCreated()); - return response; + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } + } + + private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { + List hosts = null; + if (storagePoolVO.getScope().equals(ScopeType.CLUSTER)) { + hosts = hostDao.findByClusterId(storagePoolVO.getClusterId()); + + } else if (storagePoolVO.getScope().equals(ScopeType.ZONE)) { + hosts = hostDao.findByDataCenterId(storagePoolVO.getDataCenterId()); + } + return hosts.get(0); + } + + private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd) { + String transferId = UUID.randomUUID().toString(); + int nbdPort = allocateNbdPort(); + VolumeVO volume = volumeDao.findById(cmd.getVolumeId()); + Long poolId = volume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + Host host = getFirstHostFromStoragePool(storagePoolVO); + + StartNBDServerAnswer nbdServerAnswer; + StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( + transferId, + host.getPrivateIpAddress(), + volume.getUuid(), + volume.getPath(), + nbdPort, + cmd.getDirection().toString() + ); + + try { + nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(host.getId(), nbdServerCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); } + if (!nbdServerAnswer.getResult()) { + throw new CloudRuntimeException("Failed to start the NBD server"); + } + + CreateImageTransferAnswer transferAnswer; + CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( + transferId, + host.getPrivateIpAddress(), + volume.getUuid(), + volume.getPath(), + nbdPort, + cmd.getDirection().toString() + ); + + EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId()); + transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); + + if (!transferAnswer.getResult()) { + StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, cmd.getDirection().toString(), nbdPort); + throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); + } + + ImageTransferVO imageTransfer = new ImageTransferVO( + transferId, + null, + volume.getId(), + host.getId(), + nbdPort, + ImageTransferVO.Phase.initializing, + ImageTransfer.Direction.upload, + volume.getAccountId(), + volume.getDomainId(), + volume.getDataCenterId() + ); + + imageTransfer.setTransferUrl(transferAnswer.getTransferUrl()); + imageTransfer.setSignedTicketId(transferAnswer.getImageTransferId()); + imageTransfer = imageTransferDao.persist(imageTransfer); + return imageTransfer; + } @Override - public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { - Long imageTransferId = cmd.getImageTransferId(); - - ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); - if (imageTransfer == null) { - throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); + public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { + ImageTransfer imageTransfer; + if (cmd.getDirection().equals(ImageTransfer.Direction.upload)) { + imageTransfer = createUploadImageTransfer(cmd); + } else if (cmd.getDirection().equals(ImageTransfer.Direction.download)) { + imageTransfer = createDownloadImageTransfer(cmd); + } else { + throw new CloudRuntimeException("Invalid direction: " + cmd.getDirection()); } + ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); + ImageTransferResponse response = toImageTransferResponse(imageTransferVO); + return response; + } + + private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { + + String transferId = imageTransfer.getUuid(); + int nbdPort = imageTransfer.getNbdPort(); + String direction = imageTransfer.getDirection().toString(); + FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); + BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); - FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(imageTransfer.getUuid()); try { Answer answer; if (dummyOffering) { @@ -384,17 +450,56 @@ public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { } if (!answer.getResult()) { - throw new CloudRuntimeException("Failed to create image transfer: " + answer.getDetails()); + throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); } - imageTransfer.setPhase(ImageTransferVO.Phase.finished); - imageTransferDao.update(imageTransferId, imageTransfer); - imageTransferDao.remove(imageTransferId); + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + } + } + private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { + String transferId = imageTransfer.getUuid(); + int nbdPort = imageTransfer.getNbdPort(); + String direction = imageTransfer.getDirection().toString(); + + StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); + Answer answer; + try { + answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); } + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to stop the nbd server"); + } + FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); + EndPoint ssvm = _epSelector.findSsvm(imageTransfer.getDataCenterId()); + answer = ssvm.sendMessage(finalizeCmd); + + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); + } + } + + @Override + public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { + Long imageTransferId = cmd.getImageTransferId(); + + ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); + if (imageTransfer == null) { + throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); + } + + if (imageTransfer.getDirection().equals(ImageTransfer.Direction.download)) { + finalizeDownloadImageTransfer(imageTransfer); + } else { + finalizeUploadImageTransfer(imageTransfer); + } + imageTransfer.setPhase(ImageTransferVO.Phase.finished); + imageTransferDao.update(imageTransfer.getId(), imageTransfer); + imageTransferDao.remove(imageTransfer.getId()); return true; } @@ -463,43 +568,34 @@ public List> getCommands() { return cmdList; } - // Helper methods - - private int allocateNbdPort() { - // Simplified port allocation for POC + private int getRandomNbdPort() { Random random = new Random(); return NBD_PORT_RANGE_START + random.nextInt(NBD_PORT_RANGE_END - NBD_PORT_RANGE_START); } - private String resolveDeviceName(List volumes, Long targetDiskId) { - // Simplified device name resolution for POC - int index = 0; - for (Volume vol : volumes) { - if (Long.valueOf(vol.getId()).equals(targetDiskId)) { - return "vd" + (char)('a' + index); - } - index++; + private int allocateNbdPort() { + int port = getRandomNbdPort(); + while (imageTransferDao.findByNbdPort(port) != null) { + port = getRandomNbdPort(); } - return "vda"; // fallback + return port; } - private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTransfer) { + private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTransferVO) { ImageTransferResponse response = new ImageTransferResponse(); - response.setId(imageTransfer.getUuid()); - - BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); - VMInstanceVO vm = vmInstanceDao.findById(imageTransfer.getVmId()); - Volume volume = volumeDao.findById(imageTransfer.getDiskId()); - - if (backup != null) response.setBackupId(backup.getUuid()); - if (vm != null) response.setVmId(vm.getUuid()); - if (volume != null) response.setDiskId(volume.getUuid()); - - response.setDeviceName(imageTransfer.getDeviceName()); - response.setTransferUrl(imageTransfer.getTransferUrl()); - response.setPhase(imageTransfer.getPhase().toString()); - response.setCreated(imageTransfer.getCreated()); - + response.setId(imageTransferVO.getUuid()); + Long backupId = imageTransferVO.getBackupId(); + if (backupId != null) { + Backup backup = backupDao.findById(backupId); + response.setBackupId(backup.getUuid()); + } + Long volumeId = imageTransferVO.getDiskId(); + Volume volume = volumeDao.findById(volumeId); + response.setDiskId(volume.getUuid()); + response.setTransferUrl(imageTransferVO.getTransferUrl()); + response.setPhase(ImageTransferVO.Phase.initializing.toString()); + response.setDirection(imageTransferVO.getDirection().toString()); + response.setCreated(imageTransferVO.getCreated()); return response; } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 0f8de122d883..8b3df5901590 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3716,6 +3716,93 @@ protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { return new QuerySnapshotZoneCopyAnswer(cmd, files); } + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private boolean stopImageServer() { + String unitName = "cloudstack-image-server"; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + logger.info(String.format("Image server not running, resetting failed state", unitName)); + resetService(unitName); + return true; + } + + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + logger.info(String.format("Image server %s stoppped", unitName)); + + return true; + } + + private boolean startImageServerIfNotRunning(int imageServerPort) { + final String imageServerScript = "/opt/cloud/bin/image_server.py"; + String unitName = "cloudstack-image-server"; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return true; + } + + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port 54323", + unitName, imageServerScript, imageServerPort); + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start the Image serer: %s", startResult)); + return false; + } + + // Wait with timeout until the service is up + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("Image server is now active (attempt %d)", unitName, attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + if (!serviceActive) { + logger.error(String.format("Image server failed to start within %d seconds", unitName, maxWaitSeconds)); + return false; + } + return true; + } + protected Answer execute(CreateImageTransferCommand cmd) { if (!_inSystemVM) { return new CreateImageTransferAnswer(cmd, true, "Not running inside SSVM; skipping image transfer setup."); @@ -3724,7 +3811,7 @@ protected Answer execute(CreateImageTransferCommand cmd) { final String transferId = cmd.getTransferId(); final String hostIp = cmd.getHostIpAddress(); - final String exportName = cmd.getDeviceName(); + final String exportName = cmd.getExportName(); final int nbdPort = cmd.getNbdPort(); if (StringUtils.isBlank(transferId)) { @@ -3734,15 +3821,13 @@ protected Answer execute(CreateImageTransferCommand cmd) { return new CreateImageTransferAnswer(cmd, false, "hostIpAddress is empty."); } if (StringUtils.isBlank(exportName)) { - return new CreateImageTransferAnswer(cmd, false, "deviceName is empty."); + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); } if (nbdPort <= 0) { return new CreateImageTransferAnswer(cmd, false, "Invalid nbdPort: " + nbdPort); } - final String imageServerScript = "/opt/cloud/bin/image_server.py"; final int imageServerPort = 54323; - final String imageServerLogFile = "/var/log/image_server.log"; try { // 1) Write /tmp/ with NBD endpoint details. @@ -3752,40 +3837,22 @@ protected Answer execute(CreateImageTransferCommand cmd) { payload.put("export", exportName); final String json = new GsonBuilder().create().toJson(payload); - final File transferFile = new File("/tmp", transferId); - FileUtils.writeStringToFile(transferFile, json, "UTF-8"); - - // 2) Start image_server if not already running. - final File scriptFile = new File(imageServerScript); - if (!scriptFile.exists()) { - return new CreateImageTransferAnswer(cmd, false, "Missing image server script: " + imageServerScript); + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); } + final File transferFile = new File("/tmp/imagetransfer", transferId); + FileUtils.writeStringToFile(transferFile, json, "UTF-8"); - final Script isRunning = new Script("/bin/bash", logger); - isRunning.add("-c"); - isRunning.add(String.format("pgrep -f '%s.*--port %d' >/dev/null 2>&1", imageServerScript, imageServerPort)); - final String runningResult = isRunning.execute(); - if (runningResult != null) { - try { - ProcessBuilder pb = new ProcessBuilder( - "python3", imageServerScript, - "--listen", "0.0.0.0", - "--port", String.valueOf(imageServerPort) - ); - pb.redirectOutput(ProcessBuilder.Redirect.appendTo(new File(imageServerLogFile))); - pb.redirectErrorStream(true); - pb.start(); - } catch (IOException e) { - logger.warn("Failed to start Image Server"); - return new CreateImageTransferAnswer(cmd, false, "Failed to start image server"); - } - } - final String transferUrl = String.format("http://%s:%d/images/%s", _publicIp, imageServerPort, transferId); - return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on SSVM.", transferId, transferUrl, "initializing"); - } catch (Exception e) { + } catch (IOException e) { logger.warn("Failed to prepare image transfer on SSVM", e); return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on SSVM: " + e.getMessage()); } + + startImageServerIfNotRunning(imageServerPort); + + final String transferUrl = String.format("http://%s:%d/images/%s", _publicIp, imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on SSVM.", transferId, transferUrl); } protected Answer execute(FinalizeImageTransferCommand cmd) { @@ -3798,26 +3865,17 @@ protected Answer execute(FinalizeImageTransferCommand cmd) { return new Answer(cmd, false, "transferId is empty."); } - final File transferFile = new File("/tmp", transferId); + final File transferFile = new File("/tmp/imagetransfer", transferId); if (transferFile.exists() && !transferFile.delete()) { return new Answer(cmd, false, "Failed to delete transfer config file: " + transferFile.getAbsolutePath()); } - // Stop image_server.py only if /tmp directory is empty. - final File tmpDir = new File("/tmp"); - final File[] tmpEntries = tmpDir.listFiles(); - if (tmpEntries != null && tmpEntries.length == 0) { - final String imageServerScript = "/opt/cloud/bin/image_server.py"; - final int imageServerPort = 54323; - - // Use bash "|| true" so Script returns success even if process isn't running. - final Script stop = new Script("/bin/bash", logger); - stop.add("-c"); - stop.add(String.format("pkill -f '%s.*--port %d' >/dev/null 2>&1 || true", imageServerScript, imageServerPort)); - final String stopResult = stop.execute(); - if (stopResult != null) { - return new Answer(cmd, false, "Failed to stop image server: " + stopResult); + try (Stream stream = Files.list(Paths.get("/tmp/imagetransfer"))) { + if (!stream.findAny().isPresent()) { + stopImageServer(); } + } catch (IOException e) { + logger.warn("Failed to list /tmp/imagetransfer", e); } return new Answer(cmd, true, "Image transfer finalized."); diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 2a9013fb4c5f..28513371e9d7 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -62,11 +62,11 @@ # Dynamic image_id(transferId) -> NBD export mapping: -# CloudStack writes a JSON file at /tmp/ with: +# CloudStack writes a JSON file at /tmp/imagetransfer/ with: # {"host": "...", "port": 10809, "export": "vda"} # # This server reads that file on-demand. -_CFG_DIR = "/tmp" +_CFG_DIR = "/tmp/imagetransfer" _CFG_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {} _CFG_CACHE_GUARD = threading.Lock() From aae158b2af478e342f2e607fffa645b888da5fe2 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 23 Jan 2026 09:25:57 +0530 Subject: [PATCH 011/129] upload fix-1 --- .../cloudstack/backup/CreateImageTransferCommand.java | 8 +------- .../org/apache/cloudstack/backup/ImageTransferVO.java | 2 +- .../cloudstack/backup/IncrementalBackupServiceImpl.java | 6 +++--- .../storage/resource/NfsSecondaryStorageResource.java | 8 ++++---- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index f826c01f3a62..08c06f95765f 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -23,18 +23,16 @@ public class CreateImageTransferCommand extends Command { private String transferId; private String hostIpAddress; private String exportName; - private String volumePath; private int nbdPort; private String direction; public CreateImageTransferCommand() { } - public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, String volumePath, int nbdPort, String direction) { + public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, int nbdPort, String direction) { this.transferId = transferId; this.hostIpAddress = hostIpAddress; this.exportName = exportName; - this.volumePath = volumePath; this.nbdPort = nbdPort; this.direction = direction; } @@ -60,10 +58,6 @@ public boolean executeInSequence() { return true; } - public String getVolumePath() { - return volumePath; - } - public String getDirection() { return direction; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index d9bd1f4c9ba2..4990a168a056 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -43,7 +43,7 @@ public class ImageTransferVO implements ImageTransfer { private String uuid; @Column(name = "backup_id") - private long backupId; + private Long backupId; @Column(name = "disk_id") private long diskId; diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 2eace1ff1ba9..9c5943b9e803 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -293,7 +293,6 @@ private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd) transferId, host.getPrivateIpAddress(), volume.getUuid(), - null, backup.getNbdPort(), cmd.getDirection().toString() ); @@ -354,13 +353,15 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd) { Long poolId = volume.getPoolId(); StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); Host host = getFirstHostFromStoragePool(storagePoolVO); + // todo: This only works with file based storage (not ceph, linbit) + String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); StartNBDServerAnswer nbdServerAnswer; StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, host.getPrivateIpAddress(), volume.getUuid(), - volume.getPath(), + volumePath, nbdPort, cmd.getDirection().toString() ); @@ -379,7 +380,6 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd) { transferId, host.getPrivateIpAddress(), volume.getUuid(), - volume.getPath(), nbdPort, cmd.getDirection().toString() ); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 8b3df5901590..70feb3d5a3cd 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3741,7 +3741,7 @@ private boolean stopImageServer() { stopScript.add(String.format("systemctl stop %s", unitName)); stopScript.execute(); resetService(unitName); - logger.info(String.format("Image server %s stoppped", unitName)); + logger.info(String.format("Image server %s stopped", unitName)); return true; } @@ -3759,7 +3759,7 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { } String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port 54323", + "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port %d", unitName, imageServerScript, imageServerPort); Script startScript = new Script("/bin/bash", logger); @@ -3785,7 +3785,7 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { String verifyResult = verifyScript.execute(); if (verifyResult == null) { serviceActive = true; - logger.info(String.format("Image server is now active (attempt %d)", unitName, attempt + 1)); + logger.info(String.format("Image server is now active (attempt %d)", attempt + 1)); break; } try { @@ -3797,7 +3797,7 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { } if (!serviceActive) { - logger.error(String.format("Image server failed to start within %d seconds", unitName, maxWaitSeconds)); + logger.error(String.format("Image server failed to start within %d seconds", maxWaitSeconds)); return false; } return true; From 10f65b67d7dabab78404d1cc5e03978d4bcd25d5 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 25 Jan 2026 11:39:40 +0530 Subject: [PATCH 012/129] image upload working --- .../cloudstack/backup/ImageTransfer.java | 2 +- .../cloudstack/backup/ImageTransferVO.java | 2 +- .../backup/dao/ImageTransferDao.java | 2 ++ .../backup/dao/ImageTransferDaoImpl.java | 12 +++++++ .../backup/IncrementalBackupServiceImpl.java | 27 +++++++-------- .../resource/NfsSecondaryStorageResource.java | 33 ++++++++++++++++++- 6 files changed, 62 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java index f43be2bdafe4..ca6b546e04fc 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -33,7 +33,7 @@ public enum Phase { String getUuid(); - long getBackupId(); + Long getBackupId(); long getDiskId(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 4990a168a056..25e5b213ca86 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -117,7 +117,7 @@ public String getUuid() { } @Override - public long getBackupId() { + public Long getBackupId() { return backupId; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index e5e57c4acdac..805e23d33582 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -27,4 +27,6 @@ public interface ImageTransferDao extends GenericDao { List listByBackupId(Long backupId); ImageTransferVO findByUuid(String uuid); ImageTransferVO findByNbdPort(int port); + + ImageTransferVO findByVolume(Long volumeId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 57587858661c..2a34650f2103 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -34,6 +34,7 @@ public class ImageTransferDaoImpl extends GenericDaoBase private SearchBuilder backupIdSearch; private SearchBuilder uuidSearch; private SearchBuilder nbdPortSearch; + private SearchBuilder volumeSearch; public ImageTransferDaoImpl() { } @@ -51,6 +52,10 @@ protected void init() { nbdPortSearch = createSearchBuilder(); nbdPortSearch.and("nbdPort", nbdPortSearch.entity().getNbdPort(), SearchCriteria.Op.EQ); nbdPortSearch.done(); + + volumeSearch = createSearchBuilder(); + volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ); + volumeSearch.done(); } @Override @@ -73,4 +78,11 @@ public ImageTransferVO findByNbdPort(int port) { sc.setParameters("nbdPort", port); return findOneBy(sc); } + + @Override + public ImageTransferVO findByVolume(Long volumeId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("volumeId", volumeId); + return findOneBy(sc); + } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 9c5943b9e803..5eb83516a0e7 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -273,20 +273,14 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } } - private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd) { + private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd, VolumeVO volume) { Long backupId = cmd.getBackupId(); - Long volumeId = cmd.getVolumeId(); BackupVO backup = backupDao.findById(backupId); if (backup == null) { throw new CloudRuntimeException("Backup not found: " + backupId); } boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); - Volume volume = volumeDao.findById(volumeId); - if (volume == null) { - throw new CloudRuntimeException("Volume not found: " + volumeId); - } - String transferId = UUID.randomUUID().toString(); Host host = hostDao.findById(backup.getHostId()); CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( @@ -315,7 +309,7 @@ private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd) ImageTransferVO imageTransfer = new ImageTransferVO( transferId, backupId, - volumeId, + volume.getId(), backup.getHostId(), backup.getNbdPort(), ImageTransferVO.Phase.transferring, @@ -345,17 +339,16 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { return hosts.get(0); } - private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd) { + private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd, VolumeVO volume) { String transferId = UUID.randomUUID().toString(); int nbdPort = allocateNbdPort(); - VolumeVO volume = volumeDao.findById(cmd.getVolumeId()); Long poolId = volume.getPoolId(); StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); Host host = getFirstHostFromStoragePool(storagePoolVO); + // todo: This only works with file based storage (not ceph, linbit) String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); - StartNBDServerAnswer nbdServerAnswer; StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, @@ -415,10 +408,18 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd) { @Override public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { ImageTransfer imageTransfer; + Long volumeId = cmd.getVolumeId(); + VolumeVO volume = volumeDao.findById(cmd.getVolumeId()); + + ImageTransferVO existingTransfer = imageTransferDao.findByVolume(volume.getId()); + if (existingTransfer != null) { + throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid()); + } + if (cmd.getDirection().equals(ImageTransfer.Direction.upload)) { - imageTransfer = createUploadImageTransfer(cmd); + imageTransfer = createUploadImageTransfer(cmd, volume); } else if (cmd.getDirection().equals(ImageTransfer.Direction.download)) { - imageTransfer = createDownloadImageTransfer(cmd); + imageTransfer = createDownloadImageTransfer(cmd, volume); } else { throw new CloudRuntimeException("Invalid direction: " + cmd.getDirection()); } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 70feb3d5a3cd..88b93d7643b0 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3725,14 +3725,19 @@ private void resetService(String unitName) { private boolean stopImageServer() { String unitName = "cloudstack-image-server"; + final int imageServerPort = 54323; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); String checkResult = checkScript.execute(); if (checkResult != null) { - logger.info(String.format("Image server not running, resetting failed state", unitName)); + logger.info(String.format("Image server not running, resetting failed state")); resetService(unitName); + // Still try to remove firewall rule in case it exists + if (_inSystemVM) { + removeFirewallRule(imageServerPort); + } return true; } @@ -3743,9 +3748,27 @@ private boolean stopImageServer() { resetService(unitName); logger.info(String.format("Image server %s stopped", unitName)); + // Close firewall port for image server + if (_inSystemVM) { + removeFirewallRule(imageServerPort); + } + return true; } + private void removeFirewallRule(int port) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); + Script removeScript = new Script("/bin/bash", logger); + removeScript.add("-c"); + removeScript.add(String.format("iptables -D INPUT %s || true", rule)); + String result = removeScript.execute(); + if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { + logger.debug(String.format("Firewall rule removal result for port %d: %s", port, result)); + } else { + logger.info(String.format("Firewall rule removed for port %d (or did not exist)", port)); + } + } + private boolean startImageServerIfNotRunning(int imageServerPort) { final String imageServerScript = "/opt/cloud/bin/image_server.py"; String unitName = "cloudstack-image-server"; @@ -3800,6 +3823,14 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { logger.error(String.format("Image server failed to start within %d seconds", maxWaitSeconds)); return false; } + + // Open firewall port for image server + if (_inSystemVM) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + String.format("Error in opening up image server port %d", imageServerPort)); + } + return true; } From 7b45d2e1184a3b74a6b7e9cb8f21a98e9054da8c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 27 Jan 2026 23:58:09 +0530 Subject: [PATCH 013/129] wip: changes for imagetransfer handling Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/VolumeApiService.java | 9 + .../backup/IncrementalBackupService.java | 2 + .../cloudstack/veeam/VeeamControlServlet.java | 9 +- .../veeam/adapter/UserResourceAdapter.java | 330 ++++++++++++++++++ .../veeam/api/DisksRouteHandler.java | 38 +- .../veeam/api/ImageTransfersRouteHandler.java | 126 +++++++ ...ageTransferVOToImageTransferConverter.java | 88 +++++ .../VolumeJoinVOToDiskConverter.java | 4 +- .../cloudstack/veeam/api/dto/Actions.java | 4 +- .../cloudstack/veeam/api/dto/Backup.java | 36 ++ .../apache/cloudstack/veeam/api/dto/Disk.java | 3 + .../veeam/api/dto/ImageTransfer.java | 202 +++++++++++ .../{ActionLink.java => ImageTransfers.java} | 24 +- .../{ResponseMapper.java => Mapper.java} | 4 +- .../veeam/utils/ResponseWriter.java | 4 +- .../spring-veeam-control-service-context.xml | 6 +- .../veeam/VeeamControlServiceImplTest.java | 24 ++ .../cloud/api/query/dao/VolumeJoinDao.java | 3 + .../api/query/dao/VolumeJoinDaoImpl.java | 13 + .../cloud/storage/VolumeApiServiceImpl.java | 112 +++--- .../backup/IncrementalBackupServiceImpl.java | 42 ++- .../storage/VolumeApiServiceImplTest.java | 12 +- .../resource/NfsSecondaryStorageResource.java | 7 +- 23 files changed, 980 insertions(+), 122 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/{ActionLink.java => ImageTransfers.java} (65%) rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/{ResponseMapper.java => Mapper.java} (97%) create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 1a9bcc6ee98b..b74f230d2fba 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.cloud.dc.DataCenter; import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.user.Account; @@ -70,6 +71,10 @@ public interface VolumeApiService { */ Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException; + Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name, + Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId) + throws ResourceAllocationException; + /** * Creates the volume based on the given criteria * @@ -80,6 +85,8 @@ public interface VolumeApiService { */ Volume createVolume(CreateVolumeCmd cmd); + Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display); + /** * Resizes the volume based on the given criteria * @@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId, Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); + + Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index 02c079626b4f..28f69cc38ad7 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -55,6 +55,8 @@ public interface IncrementalBackupService extends PluggableService { */ ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); + ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction); + /** * Finalize an image transfer * Marks transfer as complete (NBD is closed globally in finalize backup) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java index 7c38e4cf249e..7ebff969981d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -28,7 +28,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.utils.Negotiation; -import org.apache.cloudstack.veeam.utils.ResponseMapper; +import org.apache.cloudstack.veeam.utils.Mapper; import org.apache.cloudstack.veeam.utils.ResponseWriter; import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -38,11 +38,12 @@ public class VeeamControlServlet extends HttpServlet { private static final Logger LOGGER = LogManager.getLogger(VeeamControlServlet.class); private final ResponseWriter writer; + private final Mapper mapper; private final List routeHandlers; public VeeamControlServlet(List routeHandlers) { this.routeHandlers = routeHandlers; - ResponseMapper mapper = new ResponseMapper(); + mapper = new Mapper(); writer = new ResponseWriter(mapper); } @@ -50,6 +51,10 @@ public ResponseWriter getWriter() { return writer; } + public Mapper getMapper() { + return mapper; + } + @Override protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { String method = req.getMethod(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java new file mode 100644 index 000000000000..4be60562797c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java @@ -0,0 +1,330 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.adapter; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RolePermissionEntity; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.Rule; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; +import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; +import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.vm.StopVMCmd; +import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.backup.ImageTransfer.Direction; +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.veeam.api.converter.ImageTransferVOToImageTransferConverter; +import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.api.dto.Disk; +import org.apache.cloudstack.veeam.api.dto.ImageTransfer; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import com.cloud.api.query.dao.HostJoinDao; +import com.cloud.api.query.dao.VolumeJoinDao; +import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Grouping; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.EnumUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; + +public class UserResourceAdapter extends ManagerBase { + private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; + private static final String SERVICE_ACCOUNT_ROLE_NAME = "Veeam Service Role"; + private static final String SERVICE_ACCOUNT_FIRST_NAME = "Veeam"; + private static final String SERVICE_ACCOUNT_LAST_NAME = "Service User"; + private static final List> SERVICE_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList( + QueryAsyncJobResultCmd.class, + ListVMsCmd.class, + DeployVMCmd.class, + StartVMCmd.class, + StopVMCmd.class, + DestroyVMCmd.class, + ListVolumesCmd.class, + CreateVolumeCmd.class, + DeleteVolumeCmd.class, + AttachVolumeCmd.class, + DetachVolumeCmd.class, + ResizeVolumeCmd.class, + ListNetworksCmd.class + ); + + @Inject + DataCenterDao dataCenterDao; + + @Inject + RoleService roleService; + + @Inject + AccountService accountService; + + @Inject + AccountDao accountDao; + + @Inject + VolumeJoinDao volumeJoinDao; + + @Inject + VolumeApiService volumeApiService; + + @Inject + PrimaryDataStoreDao primaryDataStoreDao; + + @Inject + ImageTransferDao imageTransferDao; + + @Inject + HostJoinDao hostJoinDao; + + @Inject + IncrementalBackupService incrementalBackupService; + + protected Role createServiceAccountRole() { + Role role = roleService.createRole(SERVICE_ACCOUNT_ROLE_NAME, RoleType.User, + SERVICE_ACCOUNT_ROLE_NAME, false); + for (Class allowedApi : SERVICE_ACCOUNT_ROLE_ALLOWED_APIS) { + final String apiName = BaseCmd.getCommandNameByClass(allowedApi); + roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW, + String.format("Allow %s", apiName)); + } + roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY, + "Deny all"); + logger.debug("Created default role for Veeam service account in projects: {}", role); + return role; + } + + public Role getServiceAccountRole() { + List roles = roleService.findRolesByName(SERVICE_ACCOUNT_ROLE_NAME); + if (CollectionUtils.isNotEmpty(roles)) { + Role role = roles.get(0); + logger.debug("Found default role for Veeam service account in projects: {}", role); + return role; + } + return createServiceAccountRole(); + } + + protected Account createServiceAccount() { + CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); + try { + Role role = getServiceAccountRole(); + UserAccount userAccount = accountService.createUserAccount(SERVICE_ACCOUNT_NAME, + UUID.randomUUID().toString(), SERVICE_ACCOUNT_FIRST_NAME, + SERVICE_ACCOUNT_LAST_NAME, null, null, SERVICE_ACCOUNT_NAME, Account.Type.NORMAL, role.getId(), + 1L, null, null, null, null, User.Source.NATIVE); + Account account = accountService.getAccount(userAccount.getAccountId()); + logger.debug("Created Veeam service account: {}", account); + return account; + } finally { + CallContext.unregister(); + } + } + + protected Account createServiceAccountIfNeeded() { + List accounts = accountDao.findAccountsByName(SERVICE_ACCOUNT_NAME); + for (AccountVO account : accounts) { + if (Account.State.ENABLED.equals(account.getState())) { + logger.debug("Veeam service account found: {}", account); + return account; + } + } + return createServiceAccount(); + } + + @Override + public boolean start() { + createServiceAccountIfNeeded(); + //find public custom disk offering + return true; + } + + public List listAllDisks() { + List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM); + return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes); + } + + public Disk getDisk(String uuid) { + VolumeJoinVO vo = volumeJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + return VolumeJoinVOToDiskConverter.toDisk(vo); + } + + public Disk handleCreateDisk(Disk request) { + if (request == null) { + throw new InvalidParameterValueException("Request disk data is empty"); + } + String name = request.name; + if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { + throw new InvalidParameterValueException("Only worker VM disk creation is supported"); + } + if (request.storageDomains == null || CollectionUtils.isEmpty(request.storageDomains.storageDomain) || + request.storageDomains.storageDomain.size() > 1) { + throw new InvalidParameterValueException("Exactly one storage domain must be specified"); + } + Ref domain = request.storageDomains.storageDomain.get(0); + if (domain == null || domain.id == null) { + throw new InvalidParameterValueException("Storage domain ID must be specified"); + } + StoragePoolVO pool = primaryDataStoreDao.findByUuid(domain.id); + if (pool == null) { + throw new InvalidParameterValueException("Storage domain with ID " + domain.id + " not found"); + } + if (StringUtils.isBlank(request.provisionedSize)) { + throw new InvalidParameterValueException("Provisioned size must be specified"); + } + long sizeInGb; + try { + sizeInGb = Long.parseLong(request.provisionedSize); + } catch (NumberFormatException ex) { + throw new InvalidParameterValueException("Invalid provisioned size: " + request.provisionedSize); + } + if (sizeInGb <= 0) { + throw new InvalidParameterValueException("Provisioned size must be greater than zero"); + } + sizeInGb = Math.max(1L, sizeInGb / (1024L * 1024L * 1024L)); + Account serviceAccount = createServiceAccountIfNeeded(); + DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); + if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { + throw new InvalidParameterValueException("Datacenter for the specified storage domain is not found or not active"); + } + Long diskOfferingId = volumeApiService.getCustomDiskOfferingIdForVolumeUpload(serviceAccount, zone); + if (diskOfferingId == null) { + throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); + } + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + return createDisk(serviceAccount, pool, name, diskOfferingId, sizeInGb); + } finally { + CallContext.unregister(); + } + } + + @NotNull + private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb) { + Volume volume; + try { + volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, + null, name, sizeInGb, null, null, null, null); + } catch (ResourceAllocationException e) { + throw new CloudRuntimeException(e.getMessage(), e); + } + if (volume == null) { + throw new CloudRuntimeException("Failed to create volume"); + } + volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); + + // Implementation for creating a Disk resource + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId())); + } + + public List listAllImageTransfers() { + List imageTransfers = imageTransferDao.listAll(); + return ImageTransferVOToImageTransferConverter.toImageTransferList(imageTransfers, this::getHostById, this::getVolumeById); + } + + private HostJoinVO getHostById(Long hostId) { + if (hostId == null) { + return null; + } + return hostJoinDao.findById(hostId); + } + + private VolumeJoinVO getVolumeById(Long volumeId) { + if (volumeId == null) { + return null; + } + return volumeJoinDao.findById(volumeId); + } + + public ImageTransfer getImageTransfer(String uuid) { + ImageTransferVO vo = imageTransferDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); + } + return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); + } + + public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { + if (request == null) { + throw new InvalidParameterValueException("Request image transfer data is empty"); + } + if (request.getDisk() == null || StringUtils.isBlank(request.getDisk().id)) { + throw new InvalidParameterValueException("Disk ID must be specified"); + } + VolumeJoinVO volumeVO = volumeJoinDao.findByUuid(request.getDisk().id); + if (volumeVO == null) { + throw new InvalidParameterValueException("Disk with ID " + request.getDisk().id + " not found"); + } + Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); + if (direction == null) { + throw new InvalidParameterValueException("Invalid or missing direction"); + } + return createImageTransfer(null, volumeVO.getId(), direction); + } + + private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction) { + Account serviceAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + org.apache.cloudstack.backup.ImageTransfer imageTransfer = + incrementalBackupService.createImageTransfer(volumeId, null, direction); + ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); + return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); + } finally { + CallContext.unregister(); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 708daf059dbe..cf588fe23ea0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -26,22 +26,23 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.adapter.UserResourceAdapter; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.Disks; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; -import com.cloud.api.query.dao.VolumeJoinDao; -import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.fasterxml.jackson.core.JsonProcessingException; public class DisksRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/disks"; @Inject - VolumeJoinDao volumeJoinDao; + UserResourceAdapter userResourceAdapter; @Override public boolean start() { @@ -93,33 +94,32 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = VolumeJoinVOToDiskConverter.toDiskList(listDisks()); + final List result = userResourceAdapter.listAllDisks(); final Disks response = new Disks(result); - io.getWriter().write(resp, 400, response, outFormat); + io.getWriter().write(resp, 200, response, outFormat); } public void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req); logger.info("Received POST request on /api/disks endpoint, but method: POST is not supported atm. Request-data: {}", data); - - io.getWriter().write(resp, 400, "Unable to process at the moment", outFormat); - } - - protected List listDisks() { - return volumeJoinDao.listAll(); + try { + Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); + Disk response = userResourceAdapter.handleCreateDisk(request); + io.getWriter().write(resp, 201, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.getWriter().write(resp, 400, e.getMessage(), outFormat); + } } public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final VolumeJoinVO volumeJoinVO = volumeJoinDao.findByUuid(id); - if (volumeJoinVO == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; + try { + Disk response = userResourceAdapter.getDisk(id); + io.getWriter().write(resp, 200, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().write(resp, 404, e.getMessage(), outFormat); } - Disk response = VolumeJoinVOToDiskConverter.toDisk(volumeJoinVO); - - io.getWriter().write(resp, 200, response, outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java new file mode 100644 index 000000000000..58b7a418a63c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.adapter.UserResourceAdapter; +import org.apache.cloudstack.veeam.api.dto.ImageTransfer; +import org.apache.cloudstack.veeam.api.dto.ImageTransfers; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.fasterxml.jackson.core.JsonProcessingException; + +public class ImageTransfersRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/imagetransfers"; + + @Inject + UserResourceAdapter userResourceAdapter; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + if ("GET".equalsIgnoreCase(method)) { + handleGet(req, resp, outFormat, io); + return; + } + if ("POST".equalsIgnoreCase(method)) { + handlePost(req, resp, outFormat, io); + return; + } + } + + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (idAndSubPath != null) { + // /api/imagetransfers/{id} + if (idAndSubPath.first() != null) { + if (idAndSubPath.second() == null) { + handleGetById(idAndSubPath.first(), resp, outFormat, io); + return; + } + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = userResourceAdapter.listAllImageTransfers(); + final ImageTransfers response = new ImageTransfers(); + response.setImageTransfer(result); + + io.getWriter().write(resp, 400, response, outFormat); + } + + public void handlePost(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received POST request on /api/imagetransfers endpoint, but method: POST is not supported atm. Request-data: {}", data); + try { + ImageTransfer request = io.getMapper().jsonMapper().readValue(data, ImageTransfer.class); + ImageTransfer response = userResourceAdapter.handleCreateImageTransfer(request); + io.getWriter().write(resp, 201, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.getWriter().write(resp, 400, e.getMessage(), outFormat); + } + } + + public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + ImageTransfer response = userResourceAdapter.getImageTransfer(id); + io.getWriter().write(resp, 200, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().write(resp, 404, e.getMessage(), outFormat); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java new file mode 100644 index 000000000000..ff97f9469fe9 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.DisksRouteHandler; +import org.apache.cloudstack.veeam.api.HostsRouteHandler; +import org.apache.cloudstack.veeam.api.ImageTransfersRouteHandler; +import org.apache.cloudstack.veeam.api.dto.ImageTransfer; +import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Ref; + +import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.api.query.vo.VolumeJoinVO; + +public class ImageTransferVOToImageTransferConverter { + public static ImageTransfer toImageTransfer(ImageTransferVO vo, final Function hostResolver, + final Function volumeResolver) { + ImageTransfer imageTransfer = new ImageTransfer(); + final String basePath = VeeamControlService.ContextPath.value(); + imageTransfer.setId(vo.getUuid()); + imageTransfer.setHref(basePath + ImageTransfersRouteHandler.BASE_ROUTE + "/" + vo.getUuid()); + imageTransfer.setActive(Boolean.toString(true)); + imageTransfer.setDirection(vo.getDirection().name()); + imageTransfer.setFormat("cow"); + imageTransfer.setInactivityTimeout(Integer.toString(60)); + imageTransfer.setPhase(vo.getPhase().name()); + imageTransfer.setProxyUrl(vo.getTransferUrl()); + imageTransfer.setShallow(Boolean.toString(false)); + imageTransfer.setTimeoutPolicy("legacy"); + imageTransfer.setTransferUrl(vo.getTransferUrl()); + imageTransfer.setTransferred(Long.toString(0)); + if (hostResolver != null) { + HostJoinVO hostVo = hostResolver.apply(vo.getHostId()); + if (hostVo != null) { + imageTransfer.setHost(Ref.of(basePath + HostsRouteHandler.BASE_ROUTE + "/" + hostVo.getUuid(), hostVo.getUuid())); + } + } + if (volumeResolver != null) { + VolumeJoinVO volumeVo = volumeResolver.apply(vo.getDiskId()); + if (volumeVo != null) { + imageTransfer.setDisk(Ref.of(basePath + DisksRouteHandler.BASE_ROUTE + "/" + volumeVo.getUuid(), volumeVo.getUuid())); + } + } + final List links = new ArrayList<>(); + links.add(getLink(imageTransfer, "cancel")); + links.add(getLink(imageTransfer, "resume")); + links.add(getLink(imageTransfer, "pause")); + links.add(getLink(imageTransfer, "finalize")); + links.add(getLink(imageTransfer, "extend")); + return imageTransfer; + } + + public static List toImageTransferList(List vos, + final Function hostResolver, + final Function volumeResolver) { + return vos.stream().map(vo -> toImageTransfer(vo, hostResolver, volumeResolver)) + .collect(Collectors.toList()); + } + + private static Link getLink(ImageTransfer it, String rel) { + final Link link = new Link(); + link.rel = rel; + link.href = it.getHref() + "/" + rel; + return link; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 3b2305f52186..44f56bfbd008 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -97,8 +97,8 @@ public static Disk toDisk(final VolumeJoinVO vol) { // Disk profile (optional) disk.diskProfile = Ref.of( - basePath + "/diskprofiles/" + vol.getDiskOfferingId(), - String.valueOf(vol.getDiskOfferingId()) + basePath + "/diskprofiles/" + vol.getDiskOfferingUuid(), + String.valueOf(vol.getDiskOfferingUuid()) ); // Storage domains diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java index a834c579973e..9b4d0d169173 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java @@ -23,11 +23,11 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Actions { - public List link; + public List link; public Actions() {} - public Actions(final List link) { + public Actions(final List link) { this.link = link; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java new file mode 100644 index 000000000000..217a16d8131b --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +public class Backup { + + @JsonProperty("creation_date") + @JacksonXmlProperty(localName = "creation_date") + private String creationDate; + + public String getCreationDate() { + return creationDate; + } + + public void setCreationDate(String creationDate) { + this.creationDate = creationDate; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java index 812501f5615d..f61cd5d890e3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java @@ -45,6 +45,9 @@ public final class Disk { @JsonProperty("propagate_errors") public String propagateErrors; + @JsonProperty("initial_size") + public String initialSize; + @JsonProperty("provisioned_size") public String provisionedSize; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java new file mode 100644 index 000000000000..3a17b79ca05d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "image_transfer") +public class ImageTransfer { + + private String id; + private String href; + + private String active; + private String direction; + private String format; + + @JsonProperty("inactivity_timeout") + private String inactivityTimeout; + + private String phase; + + @JsonProperty("proxy_url") + private String proxyUrl; + + private String shallow; + + @JsonProperty("timeout_policy") + private String timeoutPolicy; + + @JsonProperty("transfer_url") + private String transferUrl; + + private String transferred; + + private Backup backup; + + private Ref host; + private Ref image; + private Ref disk; + private Actions actions; + + @JacksonXmlElementWrapper(useWrapping = false) + public List link; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getActive() { + return active; + } + + public void setActive(String active) { + this.active = active; + } + + public String getDirection() { + return direction; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getInactivityTimeout() { + return inactivityTimeout; + } + + public void setInactivityTimeout(String inactivityTimeout) { + this.inactivityTimeout = inactivityTimeout; + } + + public String getPhase() { + return phase; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public String getProxyUrl() { + return proxyUrl; + } + + public void setProxyUrl(String proxyUrl) { + this.proxyUrl = proxyUrl; + } + + public String getShallow() { + return shallow; + } + + public void setShallow(String shallow) { + this.shallow = shallow; + } + + public String getTimeoutPolicy() { + return timeoutPolicy; + } + + public void setTimeoutPolicy(String timeoutPolicy) { + this.timeoutPolicy = timeoutPolicy; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public String getTransferred() { + return transferred; + } + + public void setTransferred(String transferred) { + this.transferred = transferred; + } + + public Backup getBackup() { + return backup; + } + + public void setBackup(Backup backup) { + this.backup = backup; + } + + public Ref getHost() { + return host; + } + + public void setHost(Ref host) { + this.host = host; + } + + public Ref getImage() { + return image; + } + + public void setImage(Ref image) { + this.image = image; + } + + public Ref getDisk() { + return disk; + } + + public void setDisk(Ref disk) { + this.disk = disk; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java similarity index 65% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java index fe127d63364c..4414846de608 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ActionLink.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java @@ -17,23 +17,23 @@ package org.apache.cloudstack.veeam.api.dto; +import java.util.List; + import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -public final class ActionLink { - public String rel; // start/stop/reboot/shutdown... - public String href; // /api/vms/{id}/start - public String method; // "post" - - public ActionLink() {} +@JacksonXmlRootElement(localName = "image_transfers") +public class ImageTransfers { + @JsonProperty("image_transfer") + private List imageTransfer; - public ActionLink(final String rel, final String href, final String method) { - this.rel = rel; - this.href = href; - this.method = method; + public List getImageTransfer() { + return imageTransfer; } - public static ActionLink post(final String rel, final String href) { - return new ActionLink(rel, href, "post"); + public void setImageTransfer(List imageTransfer) { + this.imageTransfer = imageTransfer; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java similarity index 97% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java index 46b3a993aa72..0d6af22599e4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseMapper.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java @@ -23,11 +23,11 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -public class ResponseMapper { +public class Mapper { private final ObjectMapper json; private final XmlMapper xml; - public ResponseMapper() { + public Mapper() { this.json = new ObjectMapper(); this.xml = new XmlMapper(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java index 7dcdc3e647fc..461bb000f870 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java @@ -30,9 +30,9 @@ public final class ResponseWriter { private static final Logger LOGGER = LogManager.getLogger(ResponseWriter.class); - private final ResponseMapper mapper; + private final Mapper mapper; - public ResponseWriter(final ResponseMapper mapper) { + public ResponseWriter(final Mapper mapper) { this.mapper = mapper; } diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index e56009aacd49..1b549abcfceb 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -31,6 +31,7 @@ + @@ -39,9 +40,12 @@ - + + + + diff --git a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java new file mode 100644 index 000000000000..ee3d99fca400 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java @@ -0,0 +1,24 @@ +package org.apache.cloudstack.veeam; + +import org.apache.cloudstack.veeam.api.dto.ImageTransfer; +import org.apache.cloudstack.veeam.utils.Mapper; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import com.fasterxml.jackson.core.JsonProcessingException; + +@RunWith(MockitoJUnitRunner.class) +public class VeeamControlServiceImplTest { + + @Test + public void test_parseImageTransfer() { + String data = "{\"active\":false,\"direction\":\"upload\",\"format\":\"cow\",\"inactivity_timeout\":3600,\"phase\":\"cancelled\",\"shallow\":false,\"transferred\":0,\"link\":[],\"disk\":{\"id\":\"dba4d72d-01de-4267-aa8e-305996b53599\"},\"image\":{},\"backup\":{\"creation_date\":0}}"; + Mapper mapper = new Mapper(); + try { + ImageTransfer request = mapper.jsonMapper().readValue(data, ImageTransfer.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java index 87485e86fc9e..e61ad1d8e2d2 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.response.VolumeResponse; import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Volume; import com.cloud.utils.db.GenericDao; @@ -36,4 +37,6 @@ public interface VolumeJoinDao extends GenericDao { List searchByIds(Long... ids); List listByInstanceId(long instanceId); + + List listByHypervisor(Hypervisor.HypervisorType hypervisorType); } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 9361abef6043..0261398a2326 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -45,6 +45,7 @@ import com.cloud.user.dao.VmDiskStatisticsDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.VirtualMachine; @Component public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation implements VolumeJoinDao { @@ -379,4 +380,16 @@ public List listByInstanceId(long instanceId) { return search(sc, null); } + @Override + public List listByHypervisor(Hypervisor.HypervisorType hypervisorType) { + SearchBuilder sb = createSearchBuilder(); + sb.and("vmType", sb.entity().getVmType(), SearchCriteria.Op.EQ); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("vmType", VirtualMachine.Type.User); + sc.setParameters("hypervisorType", hypervisorType); + return search(sc, null); + } + } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 68af9750317d..3b833a1c1503 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -639,21 +639,6 @@ private Long getDefaultCustomOfferingId(Account owner, DataCenter zone) { return null; } - private Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone) { - Long offeringId = getDefaultCustomOfferingId(owner, zone); - if (offeringId != null) { - return offeringId; - } - List offerings = _diskOfferingDao.findCustomDiskOfferings(); - for (DiskOfferingVO offering : offerings) { - try { - _configMgr.checkDiskOfferingAccess(owner, offering, zone); - return offering.getId(); - } catch (PermissionDeniedException ignored) {} - } - return null; - } - @DB protected VolumeVO persistVolume(final Account owner, final Long zoneId, final String volumeName, final String url, final String format, final Long diskOfferingId, final Volume.State state) { return Transaction.execute(new TransactionCallbackWithException() { @@ -719,17 +704,31 @@ public VolumeVO doInTransaction(TransactionStatus status) { * If the retrieved volume name is null, empty or blank, then A random name * will be generated using getRandomVolumeName method. * - * @param cmd + * @param userSpecifiedName * @return Either the retrieved name or a random name. */ - public String getVolumeNameFromCommand(CreateVolumeCmd cmd) { - String userSpecifiedName = cmd.getVolumeName(); - - if (StringUtils.isBlank(userSpecifiedName)) { - userSpecifiedName = getRandomVolumeName(); + public String getVolumeNameFromCommand(String userSpecifiedName) { + if (StringUtils.isNotBlank(userSpecifiedName)) { + return userSpecifiedName; } - return userSpecifiedName; + return getRandomVolumeName(); + } + + @Override + public Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone) { + Long offeringId = getDefaultCustomOfferingId(owner, zone); + if (offeringId != null) { + return offeringId; + } + List offerings = _diskOfferingDao.findCustomDiskOfferings(); + for (DiskOfferingVO offering : offerings) { + try { + _configMgr.checkDiskOfferingAccess(owner, offering, zone); + return offering.getId(); + } catch (PermissionDeniedException ignored) {} + } + return null; } /* @@ -741,11 +740,20 @@ public String getVolumeNameFromCommand(CreateVolumeCmd cmd) { @DB @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true) public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException { + return allocVolume(cmd.getEntityOwnerId(), cmd.getZoneId(), cmd.getDiskOfferingId(), cmd.getVirtualMachineId(), + cmd.getSnapshotId(), getVolumeNameFromCommand(cmd.getVolumeName()), cmd.getSize(), + cmd.getDisplayVolume(), cmd.getMinIops(), cmd.getMaxIops(), cmd.getCustomId()); + } + + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true) + public VolumeVO allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, + String name, Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId) + throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); - long ownerId = cmd.getEntityOwnerId(); Account owner = _accountMgr.getActiveAccountById(ownerId); - Boolean displayVolume = cmd.getDisplayVolume(); // permission check _accountMgr.checkAccess(caller, null, true, _accountMgr.getActiveAccountById(ownerId)); @@ -758,8 +766,6 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept } } - Long zoneId = cmd.getZoneId(); - Long diskOfferingId = null; DiskOfferingVO diskOffering = null; Long size = null; Long minIops = null; @@ -768,13 +774,13 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept VolumeVO parentVolume = null; // validate input parameters before creating the volume - if (cmd.getSnapshotId() == null && cmd.getDiskOfferingId() == null) { + if (snapshotId == null && diskOfferingId == null) { throw new InvalidParameterValueException("At least one of disk Offering ID or snapshot ID must be passed whilst creating volume"); } // disallow passing disk offering ID with DATA disk volume snapshots - if (cmd.getSnapshotId() != null && cmd.getDiskOfferingId() != null) { - SnapshotVO snapshot = _snapshotDao.findById(cmd.getSnapshotId()); + if (snapshotId != null && diskOfferingId != null) { + SnapshotVO snapshot = _snapshotDao.findById(snapshotId); if (snapshot != null) { parentVolume = _volsDao.findByIdIncludingRemoved(snapshot.getVolumeId()); if (parentVolume != null && parentVolume.getVolumeType() != Volume.Type.ROOT) @@ -784,10 +790,8 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept } Map details = new HashMap<>(); - if (cmd.getDiskOfferingId() != null) { // create a new volume - - diskOfferingId = cmd.getDiskOfferingId(); - size = cmd.getSize(); + if (diskOfferingId != null) { // create a new volume + size = cmdSize; Long sizeInGB = size; if (size != null) { if (size > 0) { @@ -833,8 +837,8 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept if (isCustomizedIops != null) { if (isCustomizedIops) { - minIops = cmd.getMinIops(); - maxIops = cmd.getMaxIops(); + minIops = cmdMinIops; + maxIops = cmdMaxIops; if (minIops == null && maxIops == null) { minIops = 0L; @@ -866,8 +870,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept } } - if (cmd.getSnapshotId() != null) { // create volume from snapshot - Long snapshotId = cmd.getSnapshotId(); + if (snapshotId != null) { // create volume from snapshot SnapshotVO snapshotCheck = _snapshotDao.findById(snapshotId); if (snapshotCheck == null) { throw new InvalidParameterValueException("unable to find a snapshot with id " + snapshotId); @@ -918,7 +921,6 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept // one step operation - create volume in VM's cluster and attach it // to the VM - Long vmId = cmd.getVirtualMachineId(); if (vmId != null) { // Check that the virtual machine ID is valid and it's a user vm UserVmVO vm = _userVmDao.findById(vmId); @@ -960,10 +962,10 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept throw new InvalidParameterValueException("Zone is not configured to use local storage but volume's disk offering " + diskOffering.getName() + " uses it"); } - String userSpecifiedName = getVolumeNameFromCommand(cmd); + String userSpecifiedName = getVolumeNameFromCommand(name); - return commitVolume(cmd.getSnapshotId(), caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, - _uuidMgr.generateUuid(Volume.class, cmd.getCustomId()), details); + return commitVolume(snapshotId, caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, + _uuidMgr.generateUuid(Volume.class, customId), details); } @Override @@ -1075,25 +1077,33 @@ private VolumeVO allocateVolumeOnStorage(Long volumeId, Long storageId) { @DB @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", async = true) public VolumeVO createVolume(CreateVolumeCmd cmd) { - VolumeVO volume = _volsDao.findById(cmd.getEntityId()); + return createVolume(cmd.getEntityId(), cmd.getVirtualMachineId(), cmd.getSnapshotId(), cmd.getStorageId(), + cmd.getDisplayVolume()); + } + + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", async = true) + public VolumeVO createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display) { + VolumeVO volume = _volsDao.findById(volumeId); boolean created = true; try { - if (cmd.getSnapshotId() != null) { - volume = createVolumeFromSnapshot(volume, cmd.getSnapshotId(), cmd.getVirtualMachineId()); + if (snapshotId != null) { + volume = createVolumeFromSnapshot(volume, snapshotId, vmId); if (volume.getState() != Volume.State.Ready) { created = false; } // if VM Id is provided, attach the volume to the VM - if (cmd.getVirtualMachineId() != null) { + if (vmId != null) { try { - attachVolumeToVM(cmd.getVirtualMachineId(), volume.getId(), volume.getDeviceId(), false); + attachVolumeToVM(vmId, volume.getId(), volume.getDeviceId(), false); } catch (Exception ex) { StringBuilder message = new StringBuilder("Volume: "); message.append(volume.getUuid()); message.append(" created successfully, but failed to attach the newly created volume to VM: "); - message.append(cmd.getVirtualMachineId()); + message.append(vmId); message.append(" due to error: "); message.append(ex.getMessage()); if (logger.isDebugEnabled()) { @@ -1102,20 +1112,20 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { throw new CloudRuntimeException(message.toString()); } } - } else if (cmd.getStorageId() != null) { - allocateVolumeOnStorage(cmd.getEntityId(), cmd.getStorageId()); + } else if (storageId != null) { + allocateVolumeOnStorage(volumeId, storageId); } return volume; } catch (Exception e) { created = false; - VolumeInfo vol = volFactory.getVolume(cmd.getEntityId()); + VolumeInfo vol = volFactory.getVolume(volumeId); vol.stateTransit(Volume.Event.DestroyRequested); throw new CloudRuntimeException(String.format("Failed to create volume: %s", volume), e); } finally { if (!created) { VolumeVO finalVolume = volume; logger.trace("Decrementing volume resource count for account {} as volume failed to create on the backend", () -> _accountMgr.getAccount(finalVolume.getAccountId())); - _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), cmd.getDisplayVolume(), + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), display, volume.getSize(), _diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId())); } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 5eb83516a0e7..daa284274674 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -273,8 +273,8 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } } - private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd, VolumeVO volume) { - Long backupId = cmd.getBackupId(); + private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume) { + final String direction = ImageTransfer.Direction.download.toString(); BackupVO backup = backupDao.findById(backupId); if (backup == null) { throw new CloudRuntimeException("Backup not found: " + backupId); @@ -288,7 +288,7 @@ private ImageTransferVO createDownloadImageTransfer(CreateImageTransferCmd cmd, host.getPrivateIpAddress(), volume.getUuid(), backup.getNbdPort(), - cmd.getDirection().toString() + direction ); try { @@ -339,7 +339,8 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { return hosts.get(0); } - private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd, VolumeVO volume) { + private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { + final String direction = ImageTransfer.Direction.upload.toString(); String transferId = UUID.randomUUID().toString(); int nbdPort = allocateNbdPort(); @@ -356,7 +357,7 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd, Vo volume.getUuid(), volumePath, nbdPort, - cmd.getDirection().toString() + direction ); try { @@ -374,14 +375,14 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd, Vo host.getPrivateIpAddress(), volume.getUuid(), nbdPort, - cmd.getDirection().toString() + direction ); EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId()); transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); if (!transferAnswer.getResult()) { - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, cmd.getDirection().toString(), nbdPort); + StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); } @@ -407,26 +408,33 @@ private ImageTransferVO createUploadImageTransfer(CreateImageTransferCmd cmd, Vo @Override public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { + ImageTransfer imageTransfer = createImageTransfer(cmd.getVolumeId(), cmd.getBackupId(), cmd.getDirection()); + if (imageTransfer instanceof ImageTransferVO) { + ImageTransferVO imageTransferVO = (ImageTransferVO) imageTransfer; + return toImageTransferResponse(imageTransferVO); + } + return toImageTransferResponse(imageTransferDao.findById(imageTransfer.getId())); + } + + @Override + public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction) { ImageTransfer imageTransfer; - Long volumeId = cmd.getVolumeId(); - VolumeVO volume = volumeDao.findById(cmd.getVolumeId()); + VolumeVO volume = volumeDao.findById(volumeId); ImageTransferVO existingTransfer = imageTransferDao.findByVolume(volume.getId()); if (existingTransfer != null) { throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid()); } - if (cmd.getDirection().equals(ImageTransfer.Direction.upload)) { - imageTransfer = createUploadImageTransfer(cmd, volume); - } else if (cmd.getDirection().equals(ImageTransfer.Direction.download)) { - imageTransfer = createDownloadImageTransfer(cmd, volume); + if (ImageTransfer.Direction.upload.equals(direction)) { + imageTransfer = createUploadImageTransfer(volume); + } else if (ImageTransfer.Direction.download.equals(direction)) { + imageTransfer = createDownloadImageTransfer(backupId, volume); } else { - throw new CloudRuntimeException("Invalid direction: " + cmd.getDirection()); + throw new CloudRuntimeException("Invalid direction: " + direction); } - ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); - ImageTransferResponse response = toImageTransferResponse(imageTransferVO); - return response; + return imageTransferDao.findById(imageTransfer.getId()); } private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 0575b430ef10..e014ad72cfc0 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -597,26 +597,22 @@ public void testTakeSnapshotF2() throws ResourceAllocationException { @Test public void testNullGetVolumeNameFromCmd() { - when(createVol.getVolumeName()).thenReturn(null); - Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand(createVol)); + Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand(null)); } @Test public void testEmptyGetVolumeNameFromCmd() { - when(createVol.getVolumeName()).thenReturn(""); - Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand(createVol)); + Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand("")); } @Test public void testBlankGetVolumeNameFromCmd() { - when(createVol.getVolumeName()).thenReturn(" "); - Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand(createVol)); + Assert.assertNotNull(volumeApiServiceImpl.getVolumeNameFromCommand(" ")); } @Test public void testNonEmptyGetVolumeNameFromCmd() { - when(createVol.getVolumeName()).thenReturn("abc"); - Assert.assertSame(volumeApiServiceImpl.getVolumeNameFromCommand(createVol), "abc"); + Assert.assertSame(volumeApiServiceImpl.getVolumeNameFromCommand("abc"), "abc"); } @Test diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 88b93d7643b0..449525f8328e 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -54,8 +54,6 @@ import javax.naming.ConfigurationException; -import com.cloud.agent.api.ConvertSnapshotCommand; - import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.FinalizeImageTransferCommand; @@ -101,8 +99,8 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.format.ISODateTimeFormat; @@ -112,6 +110,7 @@ import com.cloud.agent.api.CheckHealthCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.ComputeChecksumCommand; +import com.cloud.agent.api.ConvertSnapshotCommand; import com.cloud.agent.api.DeleteSnapshotsDirCommand; import com.cloud.agent.api.GetStorageStatsAnswer; import com.cloud.agent.api.GetStorageStatsCommand; @@ -3827,7 +3826,7 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { // Open firewall port for image server if (_inSystemVM) { String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); - IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, String.format("Error in opening up image server port %d", imageServerPort)); } From 2350661ee34bd0c4ef87f89ac05160bac6bd4899 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 29 Jan 2026 00:27:20 +0530 Subject: [PATCH 014/129] Added progress to upload Image Transfers --- .../api/response/ImageTransferResponse.java | 8 + .../backup/IncrementalBackupService.java | 9 +- .../GetImageTransferProgressAnswer.java | 47 ++++++ .../GetImageTransferProgressCommand.java | 67 ++++++++ .../cloudstack/backup/ImageTransferVO.java | 12 ++ .../backup/dao/ImageTransferDao.java | 3 +- .../backup/dao/ImageTransferDaoImpl.java | 15 ++ .../META-INF/db/schema-42100to42200.sql | 1 + .../META-INF/db/schema-42210to42300.sql | 1 + ...etImageTransferProgressCommandWrapper.java | 102 +++++++++++++ .../backup/IncrementalBackupServiceImpl.java | 144 +++++++++++++++++- .../resource/NfsSecondaryStorageResource.java | 2 +- 12 files changed, 400 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java index 15576e8f1012..8a24ed3966f2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -62,6 +62,10 @@ public class ImageTransferResponse extends BaseResponse { @Param(description = "the image transfer direction: upload / download") private String direction; + @SerializedName("progress") + @Param(description = "progress in percentage for the upload image transfer") + private Integer progress; + @SerializedName(ApiConstants.CREATED) @Param(description = "the date created") private Date created; @@ -98,6 +102,10 @@ public void setDirection(String direction) { this.direction = direction; } + public void setProgress(Integer progress) { + this.progress = progress; + } + public void setCreated(Date created) { this.created = created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index 28f69cc38ad7..45f73a08dcf7 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -29,13 +29,20 @@ import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.CheckpointResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import com.cloud.utils.component.PluggableService; /** * Service for managing oVirt-style incremental backups using libvirt checkpoints */ -public interface IncrementalBackupService extends PluggableService { +public interface IncrementalBackupService extends Configurable, PluggableService { + + ConfigKey ImageTransferPollingInterval = new ConfigKey<>("Advanced", Long.class, + "image.transfer.polling.interval", + "10", + "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); /** * Start a backup session for a VM diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java new file mode 100644 index 000000000000..cc031abd21a0 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java @@ -0,0 +1,47 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Answer; + +public class GetImageTransferProgressAnswer extends Answer { + private Map progressMap; // transferId -> progress percentage (0-100) + + public GetImageTransferProgressAnswer() { + } + + public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details, + Map progressMap) { + super(cmd, success, details); + this.progressMap = progressMap; + } + + public Map getProgressMap() { + return progressMap; + } + + public void setProgressMap(Map progressMap) { + this.progressMap = progressMap; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java new file mode 100644 index 000000000000..2391f957f51f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java @@ -0,0 +1,67 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.List; +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class GetImageTransferProgressCommand extends Command { + private List transferIds; + private Map volumePaths; // transferId -> volume path + private Map volumeSizes; // transferId -> volume size + + public GetImageTransferProgressCommand() { + } + + public GetImageTransferProgressCommand(List transferIds, Map volumePaths, Map volumeSizes) { + this.transferIds = transferIds; + this.volumePaths = volumePaths; + this.volumeSizes = volumeSizes; + } + + public List getTransferIds() { + return transferIds; + } + + public void setTransferIds(List transferIds) { + this.transferIds = transferIds; + } + + public Map getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(Map volumePaths) { + this.volumePaths = volumePaths; + } + + public Map getVolumeSizes() { + return volumeSizes; + } + + public void setVolumeSizes(Map volumeSizes) { + this.volumeSizes = volumeSizes; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 25e5b213ca86..a6c5bce07d76 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -68,6 +68,9 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "signed_ticket_id") private String signedTicketId; + @Column(name = "progress") + private Integer progress; + @Column(name = "account_id") Long accountId; @@ -189,6 +192,15 @@ public void setSignedTicketId(String signedTicketId) { this.signedTicketId = signedTicketId; } + public Integer getProgress() { + return progress; + } + + public void setProgress(Integer progress) { + this.progress = progress; + this.updated = new Date(); + } + @Override public Class getEntityType() { return ImageTransfer.class; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index 805e23d33582..035e22958e5e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.ImageTransferVO; import com.cloud.utils.db.GenericDao; @@ -27,6 +28,6 @@ public interface ImageTransferDao extends GenericDao { List listByBackupId(Long backupId); ImageTransferVO findByUuid(String uuid); ImageTransferVO findByNbdPort(int port); - ImageTransferVO findByVolume(Long volumeId); + List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 2a34650f2103..e7d87446326c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -21,6 +21,7 @@ import javax.annotation.PostConstruct; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.ImageTransferVO; import org.springframework.stereotype.Component; @@ -35,6 +36,7 @@ public class ImageTransferDaoImpl extends GenericDaoBase private SearchBuilder uuidSearch; private SearchBuilder nbdPortSearch; private SearchBuilder volumeSearch; + private SearchBuilder phaseDirectionSearch; public ImageTransferDaoImpl() { } @@ -56,6 +58,11 @@ protected void init() { volumeSearch = createSearchBuilder(); volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ); volumeSearch.done(); + + phaseDirectionSearch = createSearchBuilder(); + phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ); + phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ); + phaseDirectionSearch.done(); } @Override @@ -85,4 +92,12 @@ public ImageTransferVO findByVolume(Long volumeId) { sc.setParameters("volumeId", volumeId); return findOneBy(sc); } + + @Override + public List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) { + SearchCriteria sc = phaseDirectionSearch.create(); + sc.setParameters("phase", phase); + sc.setParameters("direction", direction); + return listBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 858c46a7c1ee..d9f2ccd70cea 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -92,3 +92,4 @@ CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.counter', 'uc_counter__provider_ UPDATE `cloud`.`configuration` SET `scope` = 2 WHERE `name` = 'use.https.to.upload'; -- Delete the configuration for 'use.https.to.upload' from StoragePool DELETE FROM `cloud`.`storage_pool_details` WHERE `name` = 'use.https.to.upload'; + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index d3ee808cbacc..3a2bbf0bd5b1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -143,6 +143,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `progress` int COMMENT 'Transfer progress percentage (0-100)', `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', `created` datetime NOT NULL COMMENT 'date created', `updated` datetime COMMENT 'date updated if not null', diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java new file mode 100644 index 000000000000..293e87f9cefd --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java @@ -0,0 +1,102 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.backup.GetImageTransferProgressAnswer; +import org.apache.cloudstack.backup.GetImageTransferProgressCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +@ResourceWrapper(handles = GetImageTransferProgressCommand.class) +public class LibvirtGetImageTransferProgressCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(GetImageTransferProgressCommand cmd, LibvirtComputingResource resource) { + try { + List transferIds = cmd.getTransferIds(); + Map volumePaths = cmd.getVolumePaths(); + Map volumeSizes = cmd.getVolumeSizes(); + Map progressMap = new HashMap<>(); + + if (transferIds == null || transferIds.isEmpty()) { + return new GetImageTransferProgressAnswer(cmd, true, "No transfers to check", progressMap); + } + + for (String transferId : transferIds) { + String volumePath = volumePaths.get(transferId); + Long volumeSize = volumeSizes.get(transferId); + + if (volumePath == null || volumeSize == null || volumeSize == 0) { + logger.warn("Missing volume path or size for transferId: " + transferId); + progressMap.put(transferId, 0); + continue; + } + + try { + File file = new File(volumePath); + if (!file.exists()) { + logger.warn("Volume file does not exist: " + volumePath); + progressMap.put(transferId, 0); + continue; + } + + long currentSize = file.length(); + + if (volumePath.endsWith(".qcow2") || volumePath.endsWith(".qcow")) { + try { + long virtualSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); + currentSize = virtualSize; + } catch (Exception e) { + logger.warn("Failed to get virtual size for qcow2 file: " + volumePath + ", using physical size", e); + } + } + + int progress = 0; + if (volumeSize > 0) { + progress = (int) Math.min(100, Math.max(0, (currentSize * 100) / volumeSize)); + } + + progressMap.put(transferId, progress); + logger.debug("Transfer {} progress: {}% (current: {}, total: {})", transferId, progress, currentSize, volumeSize); + + } catch (Exception e) { + logger.error("Error getting progress for transferId: " + transferId + ", path: " + volumePath, e); + progressMap.put(transferId, 0); + } + } + + return new GetImageTransferProgressAnswer(cmd, true, "Progress retrieved successfully", progressMap); + + } catch (Exception e) { + logger.error("Error executing GetImageTransferProgressCommand", e); + return new GetImageTransferProgressAnswer(cmd, false, "Error getting transfer progress: " + e.getMessage()); + } + } +} diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index daa284274674..97b28468bea8 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; import java.util.UUID; import java.util.stream.Collectors; @@ -41,6 +43,8 @@ import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -96,6 +100,8 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject EndPointSelector _epSelector; + private Timer imageTransferTimer; + private static final int NBD_PORT_RANGE_START = 10809; private static final int NBD_PORT_RANGE_END = 10909; private static final boolean DATAPLANE_PROXY_MODE = true; @@ -204,7 +210,7 @@ public BackupResponse startBackup(StartBackupCmd cmd) { } catch (AgentUnavailableException | OperationTimedoutException e) { backupDao.remove(backup.getId()); - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -269,7 +275,7 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { return true; } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -324,7 +330,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu return imageTransfer; } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -363,7 +369,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(host.getId(), nbdServerCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!nbdServerAnswer.getResult()) { throw new CloudRuntimeException("Failed to start the NBD server"); @@ -392,7 +398,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { volume.getId(), host.getId(), nbdPort, - ImageTransferVO.Phase.initializing, + ImageTransferVO.Phase.transferring, ImageTransfer.Direction.upload, volume.getAccountId(), volume.getDomainId(), @@ -463,7 +469,7 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { } } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -477,7 +483,7 @@ private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { try { answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent: " + e.getMessage(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { throw new CloudRuntimeException("Failed to stop the nbd server"); @@ -602,9 +608,131 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans Volume volume = volumeDao.findById(volumeId); response.setDiskId(volume.getUuid()); response.setTransferUrl(imageTransferVO.getTransferUrl()); - response.setPhase(ImageTransferVO.Phase.initializing.toString()); + response.setPhase(imageTransferVO.getPhase().toString()); + response.setProgress(imageTransferVO.getProgress()); response.setDirection(imageTransferVO.getDirection().toString()); response.setCreated(imageTransferVO.getCreated()); return response; } + + @Override + public boolean start() { + final TimerTask imageTransferPollTask = new ManagedContextTimerTask() { + @Override + protected void runInContext() { + try { + pollImageTransferProgress(); + } catch (final Throwable t) { + logger.warn("Catch throwable in image transfer poll task ", t); + } + } + }; + + imageTransferTimer = new Timer("ImageTransferPollTask"); + long pollingInterval = ImageTransferPollingInterval.value() * 1000L; + imageTransferTimer.schedule(imageTransferPollTask, pollingInterval, pollingInterval); + return true; + } + + @Override + public boolean stop() { + if (imageTransferTimer != null) { + imageTransferTimer.cancel(); + imageTransferTimer = null; + } + return true; + } + + private void pollImageTransferProgress() { + try { + List transferringTransfers = imageTransferDao.listByPhaseAndDirection( + ImageTransfer.Phase.transferring, ImageTransfer.Direction.upload); + if (transferringTransfers == null || transferringTransfers.isEmpty()) { + return; + } + + Map> transfersByHost = transferringTransfers.stream() + .collect(Collectors.groupingBy(ImageTransferVO::getHostId)); + + for (Map.Entry> entry : transfersByHost.entrySet()) { + Long hostId = entry.getKey(); + List hostTransfers = entry.getValue(); + + try { + List transferIds = new ArrayList<>(); + Map volumePaths = new HashMap<>(); + Map volumeSizes = new HashMap<>(); + + for (ImageTransferVO transfer : hostTransfers) { + VolumeVO volume = volumeDao.findById(transfer.getDiskId()); + if (volume == null) { + logger.warn("Volume not found for image transfer: " + transfer.getUuid()); + continue; + } + + String transferId = transfer.getUuid(); + transferIds.add(transferId); + + String volumePath = volume.getPath(); + if (volumePath == null) { + logger.warn("Volume path is null for image transfer: " + transfer.getUuid()); + continue; + } + + StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); + volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), volumePath); + + volumePaths.put(transferId, volumePath); + volumeSizes.put(transferId, volume.getSize()); + } + + if (transferIds.isEmpty()) { + continue; + } + + GetImageTransferProgressCommand cmd = new GetImageTransferProgressCommand(transferIds, volumePaths, volumeSizes); + GetImageTransferProgressAnswer answer = (GetImageTransferProgressAnswer) agentManager.send(hostId, cmd); + + if (answer != null && answer.getResult() && answer.getProgressMap() != null) { + for (ImageTransferVO transfer : hostTransfers) { + String transferId = transfer.getUuid(); + Integer progress = answer.getProgressMap().get(transferId); + if (progress != null) { + transfer.setProgress(progress); + if (progress == 100) { + transfer.setPhase(ImageTransfer.Phase.finished); + logger.debug("Updated phase for image transfer {} to finished", transferId); + } + imageTransferDao.update(transfer.getId(), transfer); + logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); + } + } + } else { + logger.warn("Failed to get progress for transfers on host {}: {}", hostId, + answer != null ? answer.getDetails() : "null answer"); + } + + } catch (AgentUnavailableException | OperationTimedoutException e) { + logger.warn("Failed to communicate with host {} for image transfer progress", hostId); + } catch (Exception e) { + logger.error("Error polling image transfer progress for host " + hostId, e); + } + } + + } catch (Exception e) { + logger.error("Error in pollImageTransferProgress", e); + } + } + + @Override + public String getConfigComponentName() { + return IncrementalBackupService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + ImageTransferPollingInterval + }; + } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 449525f8328e..b3e970b0c512 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3826,7 +3826,7 @@ private boolean startImageServerIfNotRunning(int imageServerPort) { // Open firewall port for image server if (_inSystemVM) { String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); - IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, false, rule, String.format("Error in opening up image server port %d", imageServerPort)); } From 9ee97483ebd87a624acc1df9f54e45a9aaf35447 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 29 Jan 2026 08:59:35 +0530 Subject: [PATCH 015/129] get Options to return capabilities for upload --- systemvm/debian/opt/cloud/bin/image_server.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 28513371e9d7..37f457790c6d 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -51,7 +51,7 @@ # Concurrency limits across ALL images. MAX_PARALLEL_READS = 8 -MAX_PARALLEL_WRITES = 2 +MAX_PARALLEL_WRITES = 1 _READ_SEM = threading.Semaphore(MAX_PARALLEL_READS) _WRITE_SEM = threading.Semaphore(MAX_PARALLEL_WRITES) @@ -269,16 +269,7 @@ def log_message(self, fmt: str, *args: Any) -> None: def _send_imageio_headers(self) -> None: # Include these headers for compatibility with the imageio contract. - self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, PUT, OPTIONS") - self.send_header( - "Access-Control-Allow-Headers", - "Range, Content-Range, Content-Type, Content-Length", - ) - self.send_header( - "Access-Control-Expose-Headers", - "Accept-Ranges, Content-Range, Content-Length", - ) self.send_header("Accept-Ranges", "bytes") def _send_json(self, status: int, obj: Any) -> None: @@ -406,10 +397,15 @@ def do_OPTIONS(self) -> None: if self._image_cfg(image_id) is None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - self.send_response(HTTPStatus.OK) - self._send_imageio_headers() - self.send_header("Content-Length", "0") - self.end_headers() + # todo: get capabilities from backend later. this is just for upload to work + features = ["extents", "zero", "flush"] + response = { + "unix_socket": None, # Not used in this implementation + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": MAX_PARALLEL_WRITES, + } + self._send_json(HTTPStatus.OK, response) def do_GET(self) -> None: image_id, tail = self._parse_route() From f83fd00d93b82869d09eb61d75f12a70b6a8b2d8 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:04:01 +0530 Subject: [PATCH 016/129] add license to image_server.py --- systemvm/debian/opt/cloud/bin/image_server.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 37f457790c6d..440ea0593ee0 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -1,4 +1,21 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ POC "imageio-like" HTTP server backed by NBD over TCP. From 2bc311412014cc0a167ccecec949b42d744f9d0f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Jan 2026 10:19:13 +0530 Subject: [PATCH 017/129] fix precommit, license Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/VeeamControlServer.java | 2 +- .../cloudstack/veeam/VeeamControlServlet.java | 2 +- .../veeam/api/DataCentersRouteHandler.java | 2 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 2 +- .../api/converter/UserVmJoinVOToVmConverter.java | 1 - .../cloudstack/veeam/api/dto/Certificate.java | 1 - .../cloudstack/veeam/api/dto/DataCenter.java | 2 +- .../cloudstack/veeam/api/dto/DataCenters.java | 1 - .../apache/cloudstack/veeam/api/dto/Disks.java | 2 +- .../veeam/api/dto/EmptyElementSerializer.java | 1 - .../veeam/api/dto/HardwareInformation.java | 1 - .../apache/cloudstack/veeam/api/dto/Network.java | 1 - .../cloudstack/veeam/api/dto/OsVersion.java | 1 - .../cloudstack/veeam/api/dto/StorageDomains.java | 2 +- .../veeam/api/request/VmSearchExpr.java | 1 - .../cloudstack/veeam/utils/ResponseWriter.java | 1 - .../veeam-control-service/module.properties | 2 +- .../veeam/VeeamControlServiceImplTest.java | 16 ++++++++++++++++ 18 files changed, 24 insertions(+), 17 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java index 539e89e8473a..adf9e45ecdf7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -195,4 +195,4 @@ private static String dumpRequestHeaders(HttpServletRequest request) { } return sb.toString(); } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java index 7ebff969981d..69f6b9fb5c06 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -156,4 +156,4 @@ public Error(int status, String message) { public static Error badRequest(String msg) { return new Error(400, msg); } public static Error unauthorized(String msg) { return new Error(401, msg); } } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 459b076fefe7..17fece0e7ee6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -181,4 +181,4 @@ public void handleGetNetworksByDcId(final String id, final HttpServletResponse r io.getWriter().write(resp, 200, response, outFormat); } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 02c314c08eb1..08d747e955a9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -208,4 +208,4 @@ private HostJoinVO getHostById(Long hostId) { } return hostJoinDao.findById(hostId); } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 8fb2578a0286..7216eb89af12 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -156,4 +156,3 @@ private static Ref buildRef(final String baseHref, final String suffix, final St return r; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java index c95cab88de36..c90a3ea4c281 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java @@ -33,4 +33,3 @@ public class Certificate { public String getSubject() { return subject; } public void setSubject(String subject) { this.subject = subject; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java index acba378032cb..f0b8a8aff5de 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java @@ -59,4 +59,4 @@ public final class DataCenter { public String id; public DataCenter() {} -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java index 24e6f2884256..a99363a27135 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java @@ -45,4 +45,3 @@ public DataCenters(final List dataCenter) { this.dataCenter = dataCenter; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java index 302ff3adfd85..6bb2a705d444 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java @@ -37,4 +37,4 @@ public Disks() {} public Disks(final List disk) { this.disk = disk; } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java index 4b6a407aecf3..9a877d5e4b25 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java @@ -34,4 +34,3 @@ public void serialize(EmptyElement value, JsonGenerator gen, SerializerProvider gen.writeEndObject(); } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java index 83fb6d8469dd..6f2337418ee6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java @@ -48,4 +48,3 @@ public class HardwareInformation { public String getVersion() { return version; } public void setVersion(String version) { this.version = version; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java index 5c259cc8209f..0e88914141c0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java @@ -82,4 +82,3 @@ public Network() {} public String getId() { return id; } public void setId(final String id) { this.id = id; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java index 47247f91af5c..1535e0d4727c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java @@ -38,4 +38,3 @@ public class OsVersion { public String getMinor() { return minor; } public void setMinor(String minor) { this.minor = minor; } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java index 7fffa8f9a8f2..c2983bf18628 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java @@ -36,4 +36,4 @@ public StorageDomains() {} public StorageDomains(List storageDomain) { this.storageDomain = storageDomain; } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java index 56f8a38e4892..017fd9028598 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java @@ -100,4 +100,3 @@ public String toString() { } } } - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java index 461bb000f870..4b191c6c3adc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java @@ -82,4 +82,3 @@ public void writeFault(final HttpServletResponse resp, final int status, final S } } } - diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties index c444a470fb44..453e40dee69d 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. name=veeam-control-service -parent=backup \ No newline at end of file +parent=backup diff --git a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java index ee3d99fca400..4ae0808238b9 100644 --- a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java +++ b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.veeam; import org.apache.cloudstack.veeam.api.dto.ImageTransfer; From 3460a5de9989720ac569176c5a691fb47227934a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Jan 2026 17:50:51 +0530 Subject: [PATCH 018/129] veeam control changes Signed-off-by: Abhishek Kumar --- .../veeam/adapter/UserResourceAdapter.java | 31 +++- .../veeam/api/ClustersRouteHandler.java | 22 +-- .../veeam/api/DataCentersRouteHandler.java | 34 ++-- .../veeam/api/DisksRouteHandler.java | 22 +-- .../veeam/api/HostsRouteHandler.java | 20 +- .../veeam/api/ImageTransfersRouteHandler.java | 55 ++++-- .../veeam/api/NetworksRouteHandler.java | 22 +-- .../cloudstack/veeam/api/VmsRouteHandler.java | 35 ++-- .../veeam/api/VnicProfilesRouteHandler.java | 22 +-- ...ageTransferVOToImageTransferConverter.java | 15 +- .../services/PkiResourceRouteHandler.java | 173 ++++++++++++++++++ .../cloudstack/veeam/utils/PathUtil.java | 70 ++++--- .../spring-veeam-control-service-context.xml | 1 + 13 files changed, 371 insertions(+), 151 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java index 4be60562797c..ad1be6af85e9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.Rule; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; @@ -70,6 +71,7 @@ import com.cloud.org.Grouping; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; +import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.AccountVO; @@ -116,6 +118,9 @@ public class UserResourceAdapter extends ManagerBase { @Inject VolumeJoinDao volumeJoinDao; + @Inject + VolumeDetailsDao volumeDetailsDao; + @Inject VolumeApiService volumeApiService; @@ -222,19 +227,26 @@ public Disk handleCreateDisk(Disk request) { if (pool == null) { throw new InvalidParameterValueException("Storage domain with ID " + domain.id + " not found"); } - if (StringUtils.isBlank(request.provisionedSize)) { + String sizeStr = request.provisionedSize; + if (StringUtils.isBlank(sizeStr)) { throw new InvalidParameterValueException("Provisioned size must be specified"); } - long sizeInGb; + long provisionedSizeInGb; try { - sizeInGb = Long.parseLong(request.provisionedSize); + provisionedSizeInGb = Long.parseLong(sizeStr); } catch (NumberFormatException ex) { - throw new InvalidParameterValueException("Invalid provisioned size: " + request.provisionedSize); + throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); } - if (sizeInGb <= 0) { + if (provisionedSizeInGb <= 0) { throw new InvalidParameterValueException("Provisioned size must be greater than zero"); } - sizeInGb = Math.max(1L, sizeInGb / (1024L * 1024L * 1024L)); + provisionedSizeInGb = Math.max(1L, provisionedSizeInGb / (1024L * 1024L * 1024L)); + Long initialSize = null; + if (StringUtils.isNotBlank(request.initialSize)) { + try { + initialSize = Long.parseLong(request.initialSize); + } catch (NumberFormatException ignored) {} + } Account serviceAccount = createServiceAccountIfNeeded(); DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { @@ -246,14 +258,14 @@ public Disk handleCreateDisk(Disk request) { } CallContext.register(serviceAccount.getId(), serviceAccount.getId()); try { - return createDisk(serviceAccount, pool, name, diskOfferingId, sizeInGb); + return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); } finally { CallContext.unregister(); } } @NotNull - private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb) { + private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { Volume volume; try { volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, @@ -265,6 +277,9 @@ private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, throw new CloudRuntimeException("Failed to create volume"); } volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); + if (initialSize != null) { + volumeDetailsDao.addDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE, String.valueOf(initialSize), true); + } // Implementation for creating a Disk resource return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId())); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index 6459ad06f827..4c4dda45f8ce 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -31,12 +31,12 @@ import org.apache.cloudstack.veeam.api.dto.Clusters; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class ClustersRouteHandler extends ManagerBase implements RouteHandler { @@ -76,21 +76,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/clusters/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; } } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = ClusterVOToClusterConverter.toClusterList(listClusters(), this::getZoneById); final Clusters response = new Clusters(result); @@ -102,7 +100,7 @@ protected List listClusters() { return clusterDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final ClusterVO vo = clusterDao.findByUuid(id); if (vo == null) { @@ -114,7 +112,7 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } - private DataCenterJoinVO getZoneById(Long zoneId) { + protected DataCenterJoinVO getZoneById(Long zoneId) { if (zoneId == null) { return null; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 17fece0e7ee6..5c84a20bc103 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.veeam.api.dto.StorageDomains; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.dao.ImageStoreJoinDao; @@ -46,7 +47,6 @@ import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class DataCentersRouteHandler extends ManagerBase implements RouteHandler { @@ -95,20 +95,20 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/datacenters/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; + } else if (idAndSubPath.size() == 2) { + String subPath = idAndSubPath.get(1); + if ("storagedomains".equals(subPath)) { + handleGetStorageDomainsByDcId(id, resp, outFormat, io); return; } - if ("storagedomains".equals(idAndSubPath.second())) { - handleGetStorageDomainsByDcId(idAndSubPath.first(), resp, outFormat, io); - return; - } - if ("networks".equals(idAndSubPath.second())) { - handleGetNetworksByDcId(idAndSubPath.first(), resp, outFormat, io); + if ("networks".equals(subPath)) { + handleGetNetworksByDcId(id, resp, outFormat, io); return; } } @@ -117,7 +117,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = DataCenterJoinVOToDataCenterConverter.toDCList(listDCs()); final DataCenters response = new DataCenters(result); @@ -129,7 +129,7 @@ protected List listDCs() { return dataCenterJoinDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); if (dataCenterVO == null) { @@ -153,7 +153,7 @@ protected List listNetworksByDcId(final long dcId) { return networkDao.listAll(); } - public void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); if (dataCenterVO == null) { @@ -168,7 +168,7 @@ public void handleGetStorageDomainsByDcId(final String id, final HttpServletResp io.getWriter().write(resp, 200, response, outFormat); } - public void handleGetNetworksByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetNetworksByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); if (dataCenterVO == null) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index cf588fe23ea0..6cac244e1335 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -31,9 +31,9 @@ import org.apache.cloudstack.veeam.api.dto.Disks; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -78,21 +78,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.methodNotAllowed(resp, "GET", outFormat); return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/disks/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; } } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = userResourceAdapter.listAllDisks(); final Disks response = new Disks(result); @@ -100,7 +98,7 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re io.getWriter().write(resp, 200, response, outFormat); } - public void handlePost(final HttpServletRequest req, final HttpServletResponse resp, + protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req); logger.info("Received POST request on /api/disks endpoint, but method: POST is not supported atm. Request-data: {}", data); @@ -113,7 +111,7 @@ public void handlePost(final HttpServletRequest req, final HttpServletResponse r } } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { Disk response = userResourceAdapter.getDisk(id); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java index b33fa9bda9ce..6ed3a3af0b7d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -31,10 +31,10 @@ import org.apache.cloudstack.veeam.api.dto.Hosts; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.vo.HostJoinVO; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class HostsRouteHandler extends ManagerBase implements RouteHandler { @@ -71,21 +71,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/hosts/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; } } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = HostJoinVOToHostConverter.toHostList(listHosts()); final Hosts response = new Hosts(result); @@ -97,7 +95,7 @@ protected List listHosts() { return hostJoinDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final HostJoinVO vo = hostJoinDao.findByUuid(id); if (vo == null) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 58b7a418a63c..a469afc08b54 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -31,9 +31,9 @@ import org.apache.cloudstack.veeam.api.dto.ImageTransfers; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -73,17 +73,28 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } } - - if (!"GET".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET", outFormat); - return; - } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/imagetransfers/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + handleGetById(id, resp, outFormat, io); + return; + } else if (idAndSubPath.size() == 2) { + if (!"POST".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "POST", outFormat); + return; + } + String subPath = idAndSubPath.get(1); + if ("cancel".equals(subPath)) { + handleCancelById(id, resp, outFormat, io); + return; + } + if ("finalize".equals(subPath)) { + handleFinalizeById(id, resp, outFormat, io); return; } } @@ -92,7 +103,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = userResourceAdapter.listAllImageTransfers(); final ImageTransfers response = new ImageTransfers(); @@ -101,10 +112,10 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re io.getWriter().write(resp, 400, response, outFormat); } - public void handlePost(final HttpServletRequest req, final HttpServletResponse resp, + protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req); - logger.info("Received POST request on /api/imagetransfers endpoint, but method: POST is not supported atm. Request-data: {}", data); + logger.info("Received POST request on /api/imagetransfers endpoint. Request-data: {}", data); try { ImageTransfer request = io.getMapper().jsonMapper().readValue(data, ImageTransfer.class); ImageTransfer response = userResourceAdapter.handleCreateImageTransfer(request); @@ -114,7 +125,7 @@ public void handlePost(final HttpServletRequest req, final HttpServletResponse r } } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { ImageTransfer response = userResourceAdapter.getImageTransfer(id); @@ -123,4 +134,16 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 404, e.getMessage(), outFormat); } } + + protected void handleCancelById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + //ToDo: implement cancel logic + io.getWriter().write(resp, 200, "Image transfer cancelled successfully", outFormat); + } + + protected void handleFinalizeById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + //ToDo: implement finalize logic + io.getWriter().write(resp, 200, "Image transfer finalized successfully", outFormat); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index c3bab348f4e4..2b895a2a647c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -31,12 +31,12 @@ import org.apache.cloudstack.veeam.api.dto.Networks; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class NetworksRouteHandler extends ManagerBase implements RouteHandler { @@ -76,21 +76,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/networks/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; } } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = NetworkVOToNetworkConverter.toNetworkList(listNetworks(), this::getZoneById); final Networks response = new Networks(result); @@ -102,7 +100,7 @@ protected List listNetworks() { return networkDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final NetworkVO vo = networkDao.findByUuid(id); if (vo == null) { @@ -114,7 +112,7 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } - private DataCenterJoinVO getZoneById(Long zoneId) { + protected DataCenterJoinVO getZoneById(Long zoneId) { if (zoneId == null) { return null; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 08d747e955a9..6971c81b69fb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -40,13 +40,13 @@ import org.apache.cloudstack.veeam.api.response.VmEntityResponse; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.dao.VolumeJoinDao; import com.cloud.api.query.vo.HostJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class VmsRouteHandler extends ManagerBase implements RouteHandler { @@ -97,16 +97,17 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleGet(req, resp, outFormat, io); return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/vms/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } - if ("diskattachments".equals(idAndSubPath.second())) { - handleGetDisAttachmentsByVmId(idAndSubPath.first(), resp, outFormat, io); + + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; + } else if (idAndSubPath.size() == 2) { + String subPath = idAndSubPath.get(1); + if ("diskattachments".equals(subPath)) { + handleGetDisAttachmentsByVmId(id, resp, outFormat, io); return; } } @@ -115,7 +116,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final VmListQuery q = fromRequest(req); @@ -154,7 +155,7 @@ public void handleGet(final HttpServletRequest req, final HttpServletResponse re io.getWriter().write(resp, 200, response, outFormat); } - private static VmListQuery fromRequest(final HttpServletRequest req) { + protected static VmListQuery fromRequest(final HttpServletRequest req) { final VmListQuery q = new VmListQuery(); q.setSearch(req.getParameter("search")); q.setMax(parseIntOrNull(req.getParameter("max"))); @@ -162,7 +163,7 @@ private static VmListQuery fromRequest(final HttpServletRequest req) { return q; } - private static Integer parseIntOrNull(final String s) { + protected static Integer parseIntOrNull(final String s) { if (s == null || s.trim().isEmpty()) return null; try { return Integer.parseInt(s.trim()); @@ -176,7 +177,7 @@ protected List listUserVms() { return userVmJoinDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); if (userVmJoinVO == null) { @@ -188,7 +189,7 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } - public void handleGetDisAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetDisAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); if (userVmJoinVO == null) { @@ -202,7 +203,7 @@ public void handleGetDisAttachmentsByVmId(final String id, final HttpServletResp io.getWriter().write(resp, 200, response, outFormat); } - private HostJoinVO getHostById(Long hostId) { + protected HostJoinVO getHostById(Long hostId) { if (hostId == null) { return null; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index 9c2ffcca912b..ba7e040e4559 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -31,12 +31,12 @@ import org.apache.cloudstack.veeam.api.dto.VnicProfiles; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; -import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; public class VnicProfilesRouteHandler extends ManagerBase implements RouteHandler { @@ -76,21 +76,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - Pair idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); - if (idAndSubPath != null) { - // /api/vnicprofiles/{id} - if (idAndSubPath.first() != null) { - if (idAndSubPath.second() == null) { - handleGetById(idAndSubPath.first(), resp, outFormat, io); - return; - } + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; } } resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); } - public void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = NetworkVOToVnicProfileConverter.toVnicProfileList(listNetworks(), this::getZoneById); final VnicProfiles response = new VnicProfiles(result); @@ -102,7 +100,7 @@ protected List listNetworks() { return networkDao.listAll(); } - public void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { final NetworkVO vo = networkDao.findByUuid(id); if (vo == null) { @@ -114,7 +112,7 @@ public void handleGetById(final String id, final HttpServletResponse resp, final io.getWriter().write(resp, 200, response, outFormat); } - private DataCenterJoinVO getZoneById(Long zoneId) { + protected DataCenterJoinVO getZoneById(Long zoneId) { if (zoneId == null) { return null; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java index ff97f9469fe9..5fc4313bdb1c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.veeam.api.DisksRouteHandler; import org.apache.cloudstack.veeam.api.HostsRouteHandler; import org.apache.cloudstack.veeam.api.ImageTransfersRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.ImageTransfer; import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.Ref; @@ -41,11 +42,16 @@ public static ImageTransfer toImageTransfer(ImageTransferVO vo, final Function 0 && vo.getProgress() < 100)); imageTransfer.setDirection(vo.getDirection().name()); imageTransfer.setFormat("cow"); - imageTransfer.setInactivityTimeout(Integer.toString(60)); + imageTransfer.setInactivityTimeout(Integer.toString(3600)); imageTransfer.setPhase(vo.getPhase().name()); + if (org.apache.cloudstack.backup.ImageTransfer.Phase.finished.equals(vo.getPhase())) { + imageTransfer.setPhase("finished_success"); + } else if (org.apache.cloudstack.backup.ImageTransfer.Phase.failed.equals(vo.getPhase())) { + imageTransfer.setPhase("finished_failed"); + } imageTransfer.setProxyUrl(vo.getTransferUrl()); imageTransfer.setShallow(Boolean.toString(false)); imageTransfer.setTimeoutPolicy("legacy"); @@ -61,14 +67,13 @@ public static ImageTransfer toImageTransfer(ImageTransferVO vo, final Function links = new ArrayList<>(); links.add(getLink(imageTransfer, "cancel")); - links.add(getLink(imageTransfer, "resume")); - links.add(getLink(imageTransfer, "pause")); links.add(getLink(imageTransfer, "finalize")); - links.add(getLink(imageTransfer, "extend")); + imageTransfer.setActions(new Actions(links)); return imageTransfer; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java new file mode 100644 index 000000000000..19b1b88d7f34 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.services; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.Enumeration; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.component.ManagerBase; + +public class PkiResourceRouteHandler extends ManagerBase implements RouteHandler { + private static final String BASE_ROUTE = "/services/pki-resource"; + private static final String RESOURCE_KEY = "resource"; + private static final String RESOURCE_VALUE = "ca-certificate"; + private static final String FORMAT_KEY = "format"; + private static final String FORMAT_VALUE = "X509-PEM-CA"; + private static final Charset OUTPUT_CHARSET = StandardCharsets.ISO_8859_1; + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE) && "GET".equalsIgnoreCase(req.getMethod())) { + handleGet(req, resp, outFormat, io); + return; + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + protected void handleGet(HttpServletRequest req, HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + try { + final String resource = req.getParameter(RESOURCE_KEY); + final String format = req.getParameter(FORMAT_KEY); + + if (StringUtils.isNotBlank(resource) && !RESOURCE_VALUE.equals(resource)) { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Unsupported resource"); + return; + } + + if (StringUtils.isNotBlank(format) && !FORMAT_VALUE.equals(format)) { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Unsupported format"); + return; + } + + final String keystorePath = ServerPropertiesUtil.getKeystoreFile(); + final String keystorePassword = ServerPropertiesUtil.getKeystorePassword(); + + Path path = Path.of(keystorePath); + if (keystorePath.isBlank() || !Files.exists(path)) { + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "CloudStack HTTPS keystore not found"); + return; + } + + final X509Certificate caCert = + extractCaFromKeystore(path, keystorePassword); + + // DER encoding → browser downloads as .cer (oVirt behavior) + final byte[] pemBytes = + toPem(caCert).getBytes(OUTPUT_CHARSET); + resp.setStatus(HttpServletResponse.SC_OK); + resp.setHeader("Cache-Control", "no-store"); + resp.setContentType("application/x-x509-ca-cert; charset=" + OUTPUT_CHARSET.name()); + resp.setHeader("Content-Disposition", + "attachment; filename=\"pki-resource.cer\""); + resp.setContentLength(pemBytes.length); + + try (OutputStream os = resp.getOutputStream()) { + os.write(pemBytes); + } + } catch (IOException | KeyStoreException | CertificateException | NoSuchAlgorithmException e) { + String msg = "Failed to retrieve server CA certificate"; + logger.error(msg, e); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg); + } + } + + private static X509Certificate extractCaFromKeystore(Path ksPath, String ksPassword) + throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { + + final String path = ksPath.toString().toLowerCase(); + final String storeType = + (path.endsWith(".p12") || path.endsWith(".pfx")) + ? "PKCS12" + : KeyStore.getDefaultType(); + + KeyStore ks = KeyStore.getInstance(storeType); + try (var in = Files.newInputStream(ksPath)) { + ks.load(in, ksPassword != null ? ksPassword.toCharArray() : new char[0]); + } + + // Prefer HTTPS keypair alias (one with a chain) + String alias = null; + Enumeration aliases = ks.aliases(); + while (aliases.hasMoreElements()) { + String a = aliases.nextElement(); + Certificate[] chain = ks.getCertificateChain(a); + if (chain != null && chain.length > 0) { + alias = a; + break; + } + } + + if (alias == null && ks.aliases().hasMoreElements()) { + alias = ks.aliases().nextElement(); + } + + if (alias == null) { + throw new IllegalStateException("No certificate aliases in keystore"); + } + + Certificate[] chain = ks.getCertificateChain(alias); + Certificate cert = + (chain != null && chain.length > 0) + ? chain[chain.length - 1] // root-most + : ks.getCertificate(alias); + + if (!(cert instanceof X509Certificate)) { + throw new IllegalStateException("Certificate is not X509"); + } + + return (X509Certificate) cert; + } + + private static String toPem(X509Certificate cert) throws CertificateEncodingException { + String base64 = Base64.getMimeEncoder(64, new byte[]{'\n'}) + .encodeToString(cert.getEncoded()); + return "-----BEGIN CERTIFICATE-----\n" + + base64 + + "\n-----END CERTIFICATE-----\n"; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java index 11a5f2b337d3..b69748bf8bd0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java @@ -17,46 +17,58 @@ package org.apache.cloudstack.veeam.utils; -import com.cloud.utils.Pair; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.UuidUtils; public class PathUtil { - public static Pair extractIdAndSubPath(final String path, final String baseRoute) { + public static List extractIdAndSubPath(final String path, final String baseRoute) { - // baseRoute = "/api/datacenters" - if (!path.startsWith(baseRoute)) { - return null; - } + if (StringUtils.isBlank(path)) { + return null; + } - // Remove base route - String rest = path.substring(baseRoute.length()); + // Remove base route (be tolerant of trailing slash in baseRoute) + String rest = path; + if (StringUtils.isNotBlank(baseRoute)) { + String normalizedBase = baseRoute.endsWith("/") && baseRoute.length() > 1 + ? baseRoute.substring(0, baseRoute.length() - 1) + : baseRoute; + if (rest.startsWith(normalizedBase)) { + rest = rest.substring(normalizedBase.length()); + } + } - // Expect "" or "/{id}" or "/{id}/{sub}" - if (rest.isEmpty()) { - return null; // /api/datacenters (no id) - } + // Expect "/{id}" or "/{id}/..." (no empty segments) + if (StringUtils.isBlank(rest) || !rest.startsWith("/")) { + return null; // /api/datacenters (no id) or invalid format + } - if (!rest.startsWith("/")) { - return null; - } + rest = rest.substring(1); // remove leading '/' - rest = rest.substring(1); // remove leading '/' + if (StringUtils.isBlank(rest)) { + return null; + } - final String[] parts = rest.split("/", -1); + final String[] parts = rest.split("/", -1); - if (parts.length == 1) { - // /api/datacenters/{id} - if (parts[0].isEmpty()) return null; - return new Pair<>(parts[0], null); - } + // Collect non-blank segments + List validParts = new ArrayList<>(); + for (String part : parts) { + if (StringUtils.isNotBlank(part)) { + validParts.add(part); + } + } - if (parts.length == 2) { - // /api/datacenters/{id}/{subPath} - if (parts[0].isEmpty() || parts[1].isEmpty()) return null; - return new Pair<>(parts[0], parts[1]); - } + // Validate first segment is a UUID + if (validParts.isEmpty() || !UuidUtils.isUuid(validParts.get(0))) { + return null; + } - // deeper paths not handled here - return null; + return validParts; } } diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 1b549abcfceb..b247550cf140 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -32,6 +32,7 @@ + From b926c7474d8208c46f7575487843d1dea5fbfcb4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Jan 2026 17:51:19 +0530 Subject: [PATCH 019/129] server changes Signed-off-by: Abhishek Kumar --- .../GetImageTransferProgressAnswer.java | 8 +-- ...etImageTransferProgressCommandWrapper.java | 29 +++----- .../backup/IncrementalBackupServiceImpl.java | 72 +++++++++++++++---- systemvm/debian/opt/cloud/bin/image_server.py | 8 --- 4 files changed, 72 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java index cc031abd21a0..5b5713f4683a 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java @@ -22,7 +22,7 @@ import com.cloud.agent.api.Answer; public class GetImageTransferProgressAnswer extends Answer { - private Map progressMap; // transferId -> progress percentage (0-100) + private Map progressMap; // transferId -> progress percentage (0-100) public GetImageTransferProgressAnswer() { } @@ -32,16 +32,16 @@ public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boole } public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details, - Map progressMap) { + Map progressMap) { super(cmd, success, details); this.progressMap = progressMap; } - public Map getProgressMap() { + public Map getProgressMap() { return progressMap; } - public void setProgressMap(Map progressMap) { + public void setProgressMap(Map progressMap) { this.progressMap = progressMap; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java index 293e87f9cefd..7e0cbf2934db 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java @@ -43,7 +43,7 @@ public Answer execute(GetImageTransferProgressCommand cmd, LibvirtComputingResou List transferIds = cmd.getTransferIds(); Map volumePaths = cmd.getVolumePaths(); Map volumeSizes = cmd.getVolumeSizes(); - Map progressMap = new HashMap<>(); + Map progressMap = new HashMap<>(); if (transferIds == null || transferIds.isEmpty()) { return new GetImageTransferProgressAnswer(cmd, true, "No transfers to check", progressMap); @@ -54,16 +54,16 @@ public Answer execute(GetImageTransferProgressCommand cmd, LibvirtComputingResou Long volumeSize = volumeSizes.get(transferId); if (volumePath == null || volumeSize == null || volumeSize == 0) { - logger.warn("Missing volume path or size for transferId: " + transferId); - progressMap.put(transferId, 0); + logger.warn("Missing volume path or size for transferId: {}", transferId); + progressMap.put(transferId, null); continue; } try { File file = new File(volumePath); if (!file.exists()) { - logger.warn("Volume file does not exist: " + volumePath); - progressMap.put(transferId, 0); + logger.warn("Volume file does not exist: {}", volumePath); + progressMap.put(transferId, null); continue; } @@ -71,24 +71,17 @@ public Answer execute(GetImageTransferProgressCommand cmd, LibvirtComputingResou if (volumePath.endsWith(".qcow2") || volumePath.endsWith(".qcow")) { try { - long virtualSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); - currentSize = virtualSize; + currentSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); } catch (Exception e) { - logger.warn("Failed to get virtual size for qcow2 file: " + volumePath + ", using physical size", e); + logger.warn("Failed to get virtual size for qcow2 file: {}, using physical size", volumePath, e); } } - - int progress = 0; - if (volumeSize > 0) { - progress = (int) Math.min(100, Math.max(0, (currentSize * 100) / volumeSize)); - } - - progressMap.put(transferId, progress); - logger.debug("Transfer {} progress: {}% (current: {}, total: {})", transferId, progress, currentSize, volumeSize); + progressMap.put(transferId, currentSize); + logger.debug("Transfer {} progress, current: {})", transferId, currentSize, volumeSize); } catch (Exception e) { - logger.error("Error getting progress for transferId: " + transferId + ", path: " + volumePath, e); - progressMap.put(transferId, 0); + logger.error("Error getting progress for transferId: {}, path: {}", transferId, volumePath, e); + progressMap.put(transferId, null); } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 97b28468bea8..c87445afd420 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -30,6 +30,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; @@ -50,20 +51,27 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.api.ApiDBUtils; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; @@ -85,6 +93,9 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject private VolumeDao volumeDao; + @Inject + private VolumeDetailsDao volumeDetailsDao; + @Inject private AgentManager agentManager; @@ -653,6 +664,7 @@ private void pollImageTransferProgress() { Map> transfersByHost = transferringTransfers.stream() .collect(Collectors.groupingBy(ImageTransferVO::getHostId)); + Map transferVolumeMap = new HashMap<>(); for (Map.Entry> entry : transfersByHost.entrySet()) { Long hostId = entry.getKey(); @@ -669,6 +681,7 @@ private void pollImageTransferProgress() { logger.warn("Volume not found for image transfer: " + transfer.getUuid()); continue; } + transferVolumeMap.put(transfer.getId(), volume); String transferId = transfer.getUuid(); transferIds.add(transferId); @@ -693,23 +706,27 @@ private void pollImageTransferProgress() { GetImageTransferProgressCommand cmd = new GetImageTransferProgressCommand(transferIds, volumePaths, volumeSizes); GetImageTransferProgressAnswer answer = (GetImageTransferProgressAnswer) agentManager.send(hostId, cmd); - if (answer != null && answer.getResult() && answer.getProgressMap() != null) { - for (ImageTransferVO transfer : hostTransfers) { - String transferId = transfer.getUuid(); - Integer progress = answer.getProgressMap().get(transferId); - if (progress != null) { - transfer.setProgress(progress); - if (progress == 100) { - transfer.setPhase(ImageTransfer.Phase.finished); - logger.debug("Updated phase for image transfer {} to finished", transferId); - } - imageTransferDao.update(transfer.getId(), transfer); - logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); - } - } - } else { + if (answer == null || !answer.getResult() || MapUtils.isEmpty(answer.getProgressMap())) { logger.warn("Failed to get progress for transfers on host {}: {}", hostId, answer != null ? answer.getDetails() : "null answer"); + return; + } + for (ImageTransferVO transfer : hostTransfers) { + String transferId = transfer.getUuid(); + Long currentSize = answer.getProgressMap().get(transferId); + if (currentSize == null) { + continue; + } + VolumeVO volume = transferVolumeMap.get(transfer.getId()); + long totalSize = getVolumeTotalSize(volume); + int progress = Math.max((int)((currentSize * 100) / totalSize), 100); + transfer.setProgress(progress); + if (currentSize >= 100) { + transfer.setPhase(ImageTransfer.Phase.finished); + logger.debug("Updated phase for image transfer {} to finished", transferId); + } + imageTransferDao.update(transfer.getId(), transfer); + logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); } } catch (AgentUnavailableException | OperationTimedoutException e) { @@ -724,6 +741,31 @@ private void pollImageTransferProgress() { } } + private long getVolumeTotalSize(VolumeVO volume) { + VolumeDetailVO detail = volumeDetailsDao.findDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE); + if (detail != null) { + long size = NumbersUtil.parseLong(detail.getValue(), 0L); + if (size > 0) { + return size; + } + } + ApiDBUtils.getVolumeStatistics(volume.getPath()); + VolumeStats vs = null; + if (List.of(Storage.ImageFormat.VHD, Storage.ImageFormat.QCOW2, Storage.ImageFormat.RAW).contains(volume.getFormat())) { + if (volume.getPath() != null) { + vs = ApiDBUtils.getVolumeStatistics(volume.getPath()); + } + } else if (volume.getFormat() == Storage.ImageFormat.OVA) { + if (volume.getChainInfo() != null) { + vs = ApiDBUtils.getVolumeStatistics(volume.getChainInfo()); + } + } + if (vs != null && vs.getPhysicalSize() > 0) { + return vs.getPhysicalSize(); + } + return volume.getSize(); + } + @Override public String getConfigComponentName() { return IncrementalBackupService.class.getSimpleName(); diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 440ea0593ee0..7f3beb328dbd 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -586,14 +586,6 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: try: logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - size = conn.size() - if content_length != size: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"Content-Length must equal image size ({size})", - ) - return - offset = 0 remaining = content_length while remaining > 0: From da62e9a3ed8ee8c6a83263bafc5afd4f7bf98053 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Thu, 29 Jan 2026 23:15:06 +0530 Subject: [PATCH 020/129] Support multiple disks and checkpoints --- .../backup/CreateImageTransferCommand.java | 8 ++++- .../cloudstack/backup/StartBackupCommand.java | 10 +++--- .../resource/LibvirtComputingResource.java | 18 ++++++++++ .../LibvirtStartBackupCommandWrapper.java | 35 ++++++++++++------- .../backup/IncrementalBackupServiceImpl.java | 22 ++++++------ .../resource/NfsSecondaryStorageResource.java | 4 +++ 6 files changed, 68 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 08c06f95765f..43bde925f755 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -25,16 +25,18 @@ public class CreateImageTransferCommand extends Command { private String exportName; private int nbdPort; private String direction; + private String checkpointId; public CreateImageTransferCommand() { } - public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, int nbdPort, String direction) { + public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, int nbdPort, String direction, String checkpointId) { this.transferId = transferId; this.hostIpAddress = hostIpAddress; this.exportName = exportName; this.nbdPort = nbdPort; this.direction = direction; + this.checkpointId = checkpointId; } public String getExportName() { @@ -61,4 +63,8 @@ public boolean executeInSequence() { public String getDirection() { return direction; } + + public String getCheckpointId() { + return checkpointId; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index ba4daddc116d..d4ef6652b1ef 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -26,19 +26,19 @@ public class StartBackupCommand extends Command { private String toCheckpointId; private String fromCheckpointId; private int nbdPort; - private Map diskVolumePaths; // volumeId -> path mapping + private Map diskPathUuidMap; private String hostIpAddress; public StartBackupCommand() { } public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, - int nbdPort, Map diskVolumePaths, String hostIpAddress) { + int nbdPort, Map diskPathUuidMap, String hostIpAddress) { this.vmName = vmName; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; this.nbdPort = nbdPort; - this.diskVolumePaths = diskVolumePaths; + this.diskPathUuidMap = diskPathUuidMap; this.hostIpAddress = hostIpAddress; } @@ -58,8 +58,8 @@ public int getNbdPort() { return nbdPort; } - public Map getDiskVolumePaths() { - return diskVolumePaths; + public Map getDiskPathUuidMap() { + return diskPathUuidMap; } public boolean isIncremental() { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 46cf1da461e7..dc137376f7c6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -5227,6 +5227,24 @@ public void removeCheckpointsOnVm(String vmName, String volumeUuid, List logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); } + public Map getDiskPathLabelMap(String vmName) { + try { + Connect conn = LibvirtConnection.getConnectionByVmName(vmName); + List disks = getDisks(conn, vmName); + Map diskPathLabelMap = new HashMap<>(); + for (DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + diskPathLabelMap.put(disk.getDiskPath(), disk.getDiskLabel()); + } + return diskPathLabelMap; + } catch (LibvirtException e) { + logger.error("Failed to get disk path label map for VM [{}] due to: [{}].", vmName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 5013e4d79726..1dfef22c17e8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -34,6 +34,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; @ResourceWrapper(handles = StartBackupCommand.class) @@ -61,7 +62,7 @@ public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) } // Create backup XML - String backupXml = createBackupXml(cmd, fromCheckpointId, nbdPort); + String backupXml = createBackupXml(cmd, fromCheckpointId, nbdPort, resource); String checkpointXml = createCheckpointXml(toCheckpointId); // Write XMLs to temp files @@ -101,28 +102,36 @@ public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) } } - private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, int nbdPort) { + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, int nbdPort, LibvirtComputingResource resource) { StringBuilder xml = new StringBuilder(); xml.append("\n"); - if (fromCheckpointId != null && !fromCheckpointId.isEmpty()) { + if (StringUtils.isNotBlank(fromCheckpointId)) { xml.append(" ").append(fromCheckpointId).append("\n"); } - xml.append(String.format(" \n", cmd.getHostIpAddress(), nbdPort)); + xml.append(String.format(" \n", cmd.getHostIpAddress(), nbdPort)); xml.append(" \n"); - // Add disk entries - simplified for POC - Map diskPaths = cmd.getDiskVolumePaths(); - int diskIndex = 0; - for (Map.Entry entry : diskPaths.entrySet()) { - String deviceName = "vd" + (char)('a' + diskIndex); - String scratchFile = "/var/tmp/scratch-" + entry.getKey() + ".qcow2"; - xml.append(" \n"); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathLabelMap = resource.getDiskPathLabelMap(cmd.getVmName()); + + for (Map.Entry entry : diskPathLabelMap.entrySet()) { + if (!diskPathUuidMap.containsKey(entry.getKey())) { + continue; + } + String diskName = entry.getValue(); + String export = diskPathUuidMap.get(entry.getKey()); + // todo: use UUID here as well? + String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; + xml.append(" \n"); xml.append(" \n"); xml.append(" \n"); - diskIndex++; } xml.append(" \n"); diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index c87445afd420..618c7667678e 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -176,9 +176,11 @@ public BackupResponse startBackup(StartBackupCmd cmd) { backup = backupDao.persist(backup); List volumes = volumeDao.findByInstance(vmId); - Map diskVolumePaths = new HashMap<>(); + Map diskPathUuidMap = new HashMap<>(); for (Volume vol : volumes) { - diskVolumePaths.put(vol.getUuid(), vol.getPath()); + StoragePoolVO storagePool = primaryDataStoreDao.findById(vol.getPoolId()); + String volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), vol.getPath()); + diskPathUuidMap.put(volumePath, vol.getUuid()); } Host host = hostDao.findById(vm.getHostId()); @@ -187,7 +189,7 @@ public BackupResponse startBackup(StartBackupCmd cmd) { toCheckpointId, fromCheckpointId, nbdPort, - diskVolumePaths, + diskPathUuidMap, host.getPrivateIpAddress() ); @@ -240,20 +242,18 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { throw new CloudRuntimeException("Backup does not belong to VM: " + vmId); } - // Get VM VMInstanceVO vm = vmInstanceDao.findById(vmId); if (vm == null) { throw new CloudRuntimeException("VM not found: " + vmId); } - boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); + boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); List transfers = imageTransferDao.listByBackupId(backupId); if (CollectionUtils.isNotEmpty(transfers)) { throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId); } - // Send StopBackupCommand to agent StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); try { @@ -261,7 +261,7 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { if (dummyOffering) { answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); } else { - answer = (StopBackupAnswer) agentManager.send(vm.getHostId(), stopCmd); + answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); } if (!answer.getResult()) { @@ -276,7 +276,7 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { // Delete old checkpoint if exists (POC: skip actual libvirt call) if (oldCheckpointId != null) { - // In production: send command to delete oldCheckpointId via virsh checkpoint-delete + // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete logger.debug("Would delete old checkpoint: " + oldCheckpointId); } @@ -305,7 +305,8 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu host.getPrivateIpAddress(), volume.getUuid(), backup.getNbdPort(), - direction + direction, + backup.getFromCheckpointId() ); try { @@ -392,7 +393,8 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { host.getPrivateIpAddress(), volume.getUuid(), nbdPort, - direction + direction, + null ); EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId()); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index b3e970b0c512..458eb32ca890 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3865,6 +3865,10 @@ protected Answer execute(CreateImageTransferCommand cmd) { payload.put("host", hostIp); payload.put("port", nbdPort); payload.put("export", exportName); + String checkpointId = cmd.getCheckpointId(); + if (checkpointId != null) { + payload.put("export_bitmap", exportName + "-" + checkpointId.substring(0, 4)); + } final String json = new GsonBuilder().create().toJson(payload); File dir = new File("/tmp/imagetransfer"); From 4173947aa35a6b6ea0c87d7d9ccf41ef7ea9988e Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 30 Jan 2026 23:53:23 +0530 Subject: [PATCH 021/129] extents(zero/dirty) and capabilities - working todo: patch (needed?) --- systemvm/debian/opt/cloud/bin/image_server.py | 506 ++++++++++++++++-- 1 file changed, 468 insertions(+), 38 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 7f3beb328dbd..a49b2ec605a1 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -43,9 +43,12 @@ - PUT full image (Content-Length must equal export size exactly): curl -v -T demo.img http://127.0.0.1:54323/images/demo -- GET extents (POC-level; may return a single allocated extent): +- GET extents (zero/hole extents from NBD base:allocation): curl -s http://127.0.0.1:54323/images/demo/extents | jq . +- GET extents with dirty and zero (requires export_bitmap in config): + curl -s "http://127.0.0.1:54323/images/demo/extents?context=dirty" | jq . + - POST flush: curl -s -X POST http://127.0.0.1:54323/images/demo/flush | jq . """ @@ -61,11 +64,18 @@ import time from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qs import nbd CHUNK_SIZE = 256 * 1024 # 256 KiB +# NBD base:allocation flags (hole=1, zero=2; hole|zero=3) +_NBD_STATE_HOLE = 1 +_NBD_STATE_ZERO = 2 +# NBD qemu:dirty-bitmap flags (dirty=1) +_NBD_STATE_DIRTY = 1 + # Concurrency limits across ALL images. MAX_PARALLEL_READS = 8 MAX_PARALLEL_WRITES = 1 @@ -80,7 +90,7 @@ # Dynamic image_id(transferId) -> NBD export mapping: # CloudStack writes a JSON file at /tmp/imagetransfer/ with: -# {"host": "...", "port": 10809, "export": "vda"} +# {"host": "...", "port": 10809, "export": "vda", "export_bitmap":"bitmap1"} # # This server reads that file on-demand. _CFG_DIR = "/tmp/imagetransfer" @@ -92,6 +102,57 @@ def _json_bytes(obj: Any) -> bytes: return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") +def _merge_dirty_zero_extents( + allocation_extents: List[Tuple[int, int, bool]], + dirty_extents: List[Tuple[int, int, bool]], + size: int, +) -> List[Dict[str, Any]]: + """ + Merge allocation (start, length, zero) and dirty (start, length, dirty) extents + into a single list of {start, length, dirty, zero} with unified boundaries. + """ + boundaries: set[int] = {0, size} + for start, length, _ in allocation_extents: + boundaries.add(start) + boundaries.add(start + length) + for start, length, _ in dirty_extents: + boundaries.add(start) + boundaries.add(start + length) + sorted_boundaries = sorted(boundaries) + + def lookup( + extents: List[Tuple[int, int, bool]], offset: int, default: bool + ) -> bool: + for start, length, flag in extents: + if start <= offset < start + length: + return flag + return default + + result: List[Dict[str, Any]] = [] + for i in range(len(sorted_boundaries) - 1): + a, b = sorted_boundaries[i], sorted_boundaries[i + 1] + if a >= b: + continue + result.append( + { + "start": a, + "length": b - a, + "dirty": lookup(dirty_extents, a, False), + "zero": lookup(allocation_extents, a, False), + } + ) + return result + + +def _is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: + """True if extents is the single-extent fallback (dirty=false, zero=false).""" + return ( + len(extents) == 1 + and extents[0].get("dirty") is False + and extents[0].get("zero") is False + ) + + def _get_image_lock(image_id: str) -> threading.Lock: with _IMAGE_LOCKS_GUARD: lock = _IMAGE_LOCKS.get(image_id) @@ -132,7 +193,7 @@ def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: except FileNotFoundError: return None except OSError as e: - logging.warning("cfg stat failed image_id=%s err=%r", image_id, e) + logging.error("cfg stat failed image_id=%s err=%r", image_id, e) return None with _CFG_CACHE_GUARD: @@ -147,38 +208,39 @@ def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: with open(cfg_path, "rb") as f: raw = f.read(4096) except OSError as e: - logging.warning("cfg read failed image_id=%s err=%r", image_id, e) + logging.error("cfg read failed image_id=%s err=%r", image_id, e) return None try: obj = json.loads(raw.decode("utf-8")) except Exception as e: - logging.warning("cfg parse failed image_id=%s err=%r", image_id, e) + logging.error("cfg parse failed image_id=%s err=%r", image_id, e) return None if not isinstance(obj, dict): - logging.warning("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) + logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) return None host = obj.get("host") port = obj.get("port") export = obj.get("export") + export_bitmap = obj.get("export_bitmap") if not isinstance(host, str) or not host: - logging.warning("cfg missing/invalid host image_id=%s", image_id) + logging.error("cfg missing/invalid host image_id=%s", image_id) return None try: port_i = int(port) except Exception: - logging.warning("cfg missing/invalid port image_id=%s", image_id) + logging.error("cfg missing/invalid port image_id=%s", image_id) return None if port_i <= 0 or port_i > 65535: - logging.warning("cfg out-of-range port image_id=%s port=%r", image_id, port) + logging.error("cfg out-of-range port image_id=%s port=%r", image_id, port) return None if export is not None and (not isinstance(export, str) or not export): - logging.warning("cfg missing/invalid export image_id=%s", image_id) + logging.error("cfg missing/invalid export image_id=%s", image_id) return None - cfg: Dict[str, Any] = {"host": host, "port": port_i, "export": export} + cfg: Dict[str, Any] = {"host": host, "port": port_i, "export": export, "export_bitmap": export_bitmap} with _CFG_CACHE_GUARD: _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) @@ -191,7 +253,14 @@ class _NbdConn: Opens a fresh handle per request, per POC requirements. """ - def __init__(self, host: str, port: int, export: Optional[str]): + def __init__( + self, + host: str, + port: int, + export: Optional[str], + need_block_status: bool = False, + extra_meta_contexts: Optional[List[str]] = None, + ): self._sock = socket.create_connection((host, port)) self._nbd = nbd.NBD() @@ -199,6 +268,14 @@ def __init__(self, host: str, port: int, export: Optional[str]): if export and hasattr(self._nbd, "set_export_name"): self._nbd.set_export_name(export) + # Request meta contexts before connect (for block status / dirty bitmap). + if need_block_status and hasattr(self._nbd, "add_meta_context"): + for ctx in ["base:allocation"] + (extra_meta_contexts or []): + try: + self._nbd.add_meta_context(ctx) + except Exception as e: + logging.warning("add_meta_context %r failed: %r", ctx, e) + self._connect_existing_socket(self._sock) def _connect_existing_socket(self, sock: socket.socket) -> None: @@ -230,6 +307,32 @@ def _connect_existing_socket(self, sock: socket.socket) -> None: def size(self) -> int: return int(self._nbd.get_size()) + def get_capabilities(self) -> Dict[str, bool]: + """ + Query NBD export capabilities (read_only, can_flush, can_zero) from the + server handshake. Returns dict with keys read_only, can_flush, can_zero. + Uses getattr for binding name variations (is_read_only/get_read_only, etc.). + """ + out: Dict[str, bool] = { + "read_only": True, + "can_flush": False, + "can_zero": False, + } + for name, keys in [ + ("read_only", ("is_read_only", "get_read_only")), + ("can_flush", ("can_flush", "get_can_flush")), + ("can_zero", ("can_zero", "get_can_zero")), + ]: + for attr in keys: + if hasattr(self._nbd, attr): + try: + val = getattr(self._nbd, attr)() + out[name] = bool(val) + except Exception: + pass + break + return out + def pread(self, length: int, offset: int) -> bytes: # Expected signature: pread(length, offset) try: @@ -253,6 +356,235 @@ def flush(self) -> None: return raise RuntimeError("libnbd binding has no flush/fsync method") + def get_zero_extents(self) -> List[Dict[str, Any]]: + """ + Query NBD block status (base:allocation) and return extents that are + hole or zero in imageio format: [{"start": ..., "length": ..., "zero": true}, ...]. + Returns [] if block status is not supported; fallback to one full-image + zero extent when we have size but block status fails. + """ + size = self.size() + if size == 0: + return [] + + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + logging.error("get_zero_extents: no block_status/block_status_64") + return self._fallback_zero_extent(size) + if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( + "base:allocation" + ): + logging.error( + "get_zero_extents: server did not negotiate base:allocation" + ) + return self._fallback_zero_extent(size) + + zero_extents: List[Dict[str, Any]] = [] + chunk = min(size, 64 * 1024 * 1024) # 64 MiB + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + # Binding typically passes (metacontext, offset, entries[, nr_entries][, error]). + metacontext = None + off = 0 + entries = None + if len(args) >= 3: + metacontext, off, entries = args[0], args[1], args[2] + else: + for a in args: + if isinstance(a, str): + metacontext = a + elif isinstance(a, int): + off = a + elif a is not None and hasattr(a, "__iter__"): + entries = a + if metacontext != "base:allocation" or entries is None: + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + if (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0: + zero_extents.append( + {"start": current, "length": length, "zero": True} + ) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return self._fallback_zero_extent(size) + + try: + while offset < size: + count = min(chunk, size - offset) + # Try (count, offset, callback) then (offset, count, callback) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.error("get_zero_extents block_status failed: %r", e) + return self._fallback_zero_extent(size) + if not zero_extents: + return self._fallback_zero_extent(size) + return zero_extents + + def _fallback_zero_extent(self, size: int) -> List[Dict[str, Any]]: + """Return one zero extent covering the whole image when block status unavailable.""" + return [{"start": 0, "length": size, "zero": True}] + + def get_allocation_extents(self) -> List[Dict[str, Any]]: + """ + Query base:allocation and return all extents (allocated and hole/zero) + as [{"start": ..., "length": ..., "zero": bool}, ...]. + Fallback when block status unavailable: one extent with zero=False. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return [{"start": 0, "length": size, "zero": False}] + if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( + "base:allocation" + ): + return [{"start": 0, "length": size, "zero": False}] + + allocation_extents: List[Dict[str, Any]] = [] + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if metacontext != "base:allocation" or entries is None: + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 + allocation_extents.append( + {"start": current, "length": length, "zero": zero} + ) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return [{"start": 0, "length": size, "zero": False}] + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_allocation_extents block_status failed: %r", e) + return [{"start": 0, "length": size, "zero": False}] + if not allocation_extents: + return [{"start": 0, "length": size, "zero": False}] + return allocation_extents + + def get_extents_dirty_and_zero( + self, dirty_bitmap_context: str + ) -> List[Dict[str, Any]]: + """ + Query block status for base:allocation and qemu:dirty-bitmap:, + merge boundaries, and return extents with dirty and zero flags. + Format: [{"start": ..., "length": ..., "dirty": bool, "zero": bool}, ...]. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return self._fallback_dirty_zero_extents(size) + if hasattr(self._nbd, "can_meta_context"): + if not self._nbd.can_meta_context("base:allocation"): + return self._fallback_dirty_zero_extents(size) + if not self._nbd.can_meta_context(dirty_bitmap_context): + logging.warning( + "dirty bitmap context %r not negotiated", dirty_bitmap_context + ) + return self._fallback_dirty_zero_extents(size) + + allocation_extents: List[Tuple[int, int, bool]] = [] # (start, length, zero) + dirty_extents: List[Tuple[int, int, bool]] = [] # (start, length, dirty) + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if entries is None or not hasattr(entries, "__iter__"): + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + if metacontext == "base:allocation": + zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 + allocation_extents.append((current, length, zero)) + elif metacontext == dirty_bitmap_context: + dirty = (flags & _NBD_STATE_DIRTY) != 0 + dirty_extents.append((current, length, dirty)) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return self._fallback_dirty_zero_extents(size) + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_extents_dirty_and_zero block_status failed: %r", e) + return self._fallback_dirty_zero_extents(size) + return _merge_dirty_zero_extents(allocation_extents, dirty_extents, size) + + def _fallback_dirty_zero_extents(self, size: int) -> List[Dict[str, Any]]: + """One extent: whole image, dirty=false, zero=false when bitmap unavailable.""" + return [{"start": 0, "length": size, "dirty": False, "zero": False}] + def close(self) -> None: # Best-effort; bindings may differ. try: @@ -284,15 +616,24 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, fmt: str, *args: Any) -> None: logging.info("%s - - %s", self.address_string(), fmt % args) - def _send_imageio_headers(self) -> None: + def _send_imageio_headers( + self, allowed_methods: Optional[str] = None + ) -> None: # Include these headers for compatibility with the imageio contract. - self.send_header("Access-Control-Allow-Methods", "GET, PUT, OPTIONS") + if allowed_methods is None: + allowed_methods = "GET, PUT, OPTIONS" + self.send_header("Access-Control-Allow-Methods", allowed_methods) self.send_header("Accept-Ranges", "bytes") - def _send_json(self, status: int, obj: Any) -> None: + def _send_json( + self, + status: int, + obj: Any, + allowed_methods: Optional[str] = None, + ) -> None: body = _json_bytes(obj) self.send_response(status) - self._send_imageio_headers() + self._send_imageio_headers(allowed_methods) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() @@ -403,6 +744,13 @@ def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: return None, None return image_id, tail + def _parse_query(self) -> Dict[str, List[str]]: + """Parse query string from self.path into a dict of name -> list of values.""" + if "?" not in self.path: + return {} + query = self.path.split("?", 1)[1] + return parse_qs(query, keep_blank_values=True) + def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: return _load_image_cfg(image_id) @@ -411,18 +759,50 @@ def do_OPTIONS(self) -> None: if image_id is None or tail is not None: self._send_error_json(HTTPStatus.NOT_FOUND, "not found") return - if self._image_cfg(image_id) is None: + cfg = self._image_cfg(image_id) + if cfg is None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - # todo: get capabilities from backend later. this is just for upload to work - features = ["extents", "zero", "flush"] + # Query NBD backend for capabilities (like nbdinfo); fall back to config. + read_only = True + can_flush = False + can_zero = False + try: + with _NbdConn( + cfg["host"], + int(cfg["port"]), + cfg.get("export"), + ) as conn: + caps = conn.get_capabilities() + read_only = caps["read_only"] + can_flush = caps["can_flush"] + can_zero = caps["can_zero"] + except Exception as e: + logging.warning("OPTIONS: could not query NBD capabilities: %r", e) + read_only = bool(cfg.get("read_only")) + if not read_only: + can_flush = True + can_zero = True + # Report options for this image from NBD: read-only => no PUT; only advertise supported features. + if read_only: + allowed_methods = "GET, OPTIONS" + features = ["extents"] + max_writers = 0 + else: + allowed_methods = "GET, PUT, OPTIONS" + features = ["extents"] + if can_zero: + features.append("zero") + if can_flush: + features.append("flush") + max_writers = MAX_PARALLEL_WRITES if not read_only else 0 response = { "unix_socket": None, # Not used in this implementation "features": features, "max_readers": MAX_PARALLEL_READS, - "max_writers": MAX_PARALLEL_WRITES, + "max_writers": max_writers, } - self._send_json(HTTPStatus.OK, response) + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) def do_GET(self) -> None: image_id, tail = self._parse_route() @@ -436,7 +816,9 @@ def do_GET(self) -> None: return if tail == "extents": - self._handle_get_extents(image_id, cfg) + query = self._parse_query() + context = (query.get("context") or [None])[0] + self._handle_get_extents(image_id, cfg, context=context) return if tail is not None: self._send_error_json(HTTPStatus.NOT_FOUND, "not found") @@ -556,7 +938,7 @@ def _handle_get_image( bytes_sent += len(data) except Exception as e: # If headers already sent, we can't return JSON reliably; just log. - logging.warning("GET error image_id=%s err=%r", image_id, e) + logging.error("GET error image_id=%s err=%r", image_id, e) try: if not self.wfile.closed: self.close_connection = True @@ -604,7 +986,7 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) except Exception as e: - logging.warning("PUT error image_id=%s err=%r", image_id, e) + logging.error("PUT error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: _WRITE_SEM.release() @@ -614,10 +996,11 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur ) - def _handle_get_extents(self, image_id: str, cfg: Dict[str, Any]) -> None: - # Keep deterministic and simple (POC): report entire image allocated. - # No per-image lock required by spec, but we still take it to avoid racing - # with a write and to keep behavior consistent. + def _handle_get_extents( + self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None + ) -> None: + # context=dirty: return extents with dirty and zero from base:allocation + bitmap. + # Otherwise: return zero/hole extents from base:allocation only. lock = _get_image_lock(image_id) if not lock.acquire(blocking=False): self._send_error_json(HTTPStatus.CONFLICT, "image busy") @@ -625,15 +1008,62 @@ def _handle_get_extents(self, image_id: str, cfg: Dict[str, Any]) -> None: start = _now_s() try: - logging.info("EXTENTS start image_id=%s", image_id) - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - size = conn.size() - self._send_json( - HTTPStatus.OK, - [{"start": 0, "length": size, "allocated": True}], - ) + logging.info("EXTENTS start image_id=%s context=%s", image_id, context) + if context == "dirty": + export_bitmap = cfg.get("export_bitmap") + if not export_bitmap: + # Fallback: same structure as zero extents but dirty=true for all ranges + with _NbdConn( + cfg["host"], + int(cfg["port"]), + cfg.get("export"), + need_block_status=True, + ) as conn: + allocation = conn.get_allocation_extents() + extents = [ + {"start": e["start"], "length": e["length"], "dirty": True, "zero": e["zero"]} + for e in allocation + ] + else: + dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}" + extra_contexts: List[str] = [dirty_bitmap_ctx] + with _NbdConn( + cfg["host"], + int(cfg["port"]), + cfg.get("export"), + need_block_status=True, + extra_meta_contexts=extra_contexts, + ) as conn: + extents = conn.get_extents_dirty_and_zero(dirty_bitmap_ctx) + # When bitmap not actually available, same fallback: zero structure + dirty=true + if _is_fallback_dirty_response(extents): + with _NbdConn( + cfg["host"], + int(cfg["port"]), + cfg.get("export"), + need_block_status=True, + ) as conn: + allocation = conn.get_allocation_extents() + extents = [ + { + "start": e["start"], + "length": e["length"], + "dirty": True, + "zero": e["zero"], + } + for e in allocation + ] + else: + with _NbdConn( + cfg["host"], + int(cfg["port"]), + cfg.get("export"), + need_block_status=True, + ) as conn: + extents = conn.get_zero_extents() + self._send_json(HTTPStatus.OK, extents) except Exception as e: - logging.warning("EXTENTS error image_id=%s err=%r", image_id, e) + logging.error("EXTENTS error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: lock.release() @@ -653,7 +1083,7 @@ def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: conn.flush() self._send_json(HTTPStatus.OK, {"ok": True}) except Exception as e: - logging.warning("FLUSH error image_id=%s err=%r", image_id, e) + logging.error("FLUSH error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: lock.release() From 91a081beece94b55783738f749255c1383110c2f Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 2 Feb 2026 08:28:37 +0530 Subject: [PATCH 022/129] Patch (zero, data) + Flush support in image_server.py --- systemvm/debian/opt/cloud/bin/image_server.py | 289 +++++++++++++++++- 1 file changed, 287 insertions(+), 2 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index a49b2ec605a1..848eb41983c0 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -27,7 +27,7 @@ apt install python3-libnbd - Run server: - python image_server.py --listen 0.0.0.0 --port 54323 + createImageTransfer will start the server as a systemd service 'cloudstack-image-server' Example curl commands -------------------- @@ -51,6 +51,34 @@ - POST flush: curl -s -X POST http://127.0.0.1:54323/images/demo/flush | jq . + +- PATCH zero (zero a byte range; application/json body): + curl -k -X PATCH \ + -H "Content-Type: application/json" \ + --data-binary '{"op": "zero", "offset": 4096, "size": 8192}' \ + http://127.0.0.1:54323/images/demo + + Zero at offset 1 GiB, 4096 bytes, no flush: + curl -k -X PATCH \ + -H "Content-Type: application/json" \ + --data-binary '{"op": "zero", "offset": 1073741824, "size": 4096}' \ + http://127.0.0.1:54323/images/demo + + Zero entire disk and flush: + curl -k -X PATCH \ + -H "Content-Type: application/json" \ + --data-binary '{"op": "zero", "size": 107374182400, "flush": true}' \ + http://127.0.0.1:54323/images/demo + +- PATCH flush (flush data to storage; operates on entire image): + curl -k -X PATCH \ + -H "Content-Type: application/json" \ + --data-binary '{"op": "flush"}' \ + http://127.0.0.1:54323/images/demo + +- PATCH range (write binary body at byte range; Range + Content-Length required): + curl -v -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin \ + http://127.0.0.1:54323/images/demo """ from __future__ import annotations @@ -347,6 +375,37 @@ def pwrite(self, buf: bytes, offset: int) -> None: except TypeError: # pragma: no cover (binding differences) self._nbd.pwrite(offset, buf) + def pzero(self, offset: int, size: int) -> None: + """ + Zero a byte range. Uses NBD WRITE_ZEROES when available (efficient/punch hole), + otherwise falls back to writing zero bytes via pwrite. + """ + if size <= 0: + return + # Try libnbd pwrite_zeros / zero; argument order varies by binding. + for name in ("pwrite_zeros", "zero"): + if not hasattr(self._nbd, name): + continue + fn = getattr(self._nbd, name) + try: + fn(size, offset) + return + except TypeError: + try: + fn(offset, size) + return + except TypeError: + pass + # Fallback: write zeros in chunks. + remaining = size + pos = offset + zero_buf = b"\x00" * min(CHUNK_SIZE, size) + while remaining > 0: + chunk = min(len(zero_buf), remaining) + self.pwrite(zero_buf[:chunk], pos) + pos += chunk + remaining -= chunk + def flush(self) -> None: if hasattr(self._nbd, "flush"): self._nbd.flush() @@ -789,7 +848,8 @@ def do_OPTIONS(self) -> None: features = ["extents"] max_writers = 0 else: - allowed_methods = "GET, PUT, OPTIONS" + # PATCH: JSON (zero/flush) and Range+binary (write byte range). + allowed_methods = "GET, PUT, PATCH, OPTIONS" features = ["extents"] if can_zero: features.append("zero") @@ -875,6 +935,111 @@ def do_POST(self) -> None: return self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + def do_PATCH(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + range_header = self.headers.get("Range") + + # Binary PATCH: Range + body writes bytes at that range (e.g. curl -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin). + if range_header is not None and content_type != "application/json": + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") + return + self._handle_patch_range(image_id, cfg, range_header, content_length) + return + + # JSON PATCH: application/json with op (zero, flush). + if content_type != "application/json": + self._send_error_json( + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0 or content_length > 64 * 1024: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + body = self.rfile.read(content_length) + if len(body) != content_length: + self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") + return + + try: + payload = json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") + return + + if not isinstance(payload, dict): + self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") + return + + op = payload.get("op") + if op == "flush": + # Flush entire image; offset and size are ignored (per spec). + self._handle_post_flush(image_id, cfg) + return + if op != "zero": + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "unsupported op; only \"zero\" and \"flush\" are supported", + ) + return + + try: + size = int(payload.get("size")) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") + return + if size <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") + return + + offset = payload.get("offset") + if offset is None: + offset = 0 + else: + try: + offset = int(offset) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") + return + if offset < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") + return + + flush = bool(payload.get("flush", False)) + + self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) + def _handle_get_image( self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] ) -> None: @@ -1090,6 +1255,126 @@ def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: dur = _now_s() - start logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) + def _handle_patch_zero( + self, + image_id: str, + cfg: Dict[str, Any], + offset: int, + size: int, + flush: bool, + ) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + try: + logging.info( + "PATCH zero start image_id=%s offset=%d size=%d flush=%s", + image_id, offset, size, flush, + ) + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + image_size = conn.size() + if offset >= image_size: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "offset must be less than image size", + ) + return + zero_size = min(size, image_size - offset) + conn.pzero(offset, zero_size) + if flush: + conn.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + except Exception as e: + logging.error("PATCH zero error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_patch_range( + self, + image_id: str, + cfg: Dict[str, Any], + range_header: str, + content_length: int, + ) -> None: + """Write request body to the image at the byte range from Range header.""" + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + bytes_written = 0 + try: + logging.info( + "PATCH range start image_id=%s range=%s content_length=%d", + image_id, range_header, content_length, + ) + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + image_size = conn.size() + try: + start_off, end_inclusive = self._parse_single_range( + range_header, image_size + ) + except ValueError as e: + if "unsatisfiable" in str(e).lower(): + self._send_range_not_satisfiable(image_size) + else: + self._send_error_json( + HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" + ) + return + expected_len = end_inclusive - start_off + 1 + if content_length != expected_len: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"Content-Length ({content_length}) must equal range length ({expected_len})", + ) + return + offset = start_off + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {bytes_written} bytes", + ) + return + conn.pwrite(chunk, offset) + n = len(chunk) + offset += n + remaining -= n + bytes_written += n + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + except Exception as e: + logging.error("PATCH range error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "PATCH range end image_id=%s bytes=%d duration_s=%.3f", + image_id, bytes_written, dur, + ) + def main() -> None: parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") From c36cd2c26cb5d1a171ecae96ee317d9ccb234d13 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 5 Feb 2026 09:12:50 +0530 Subject: [PATCH 023/129] Backup of stopped VMs --- .../cloudstack/backup/StartBackupCommand.java | 16 +- .../LibvirtStartBackupCommandWrapper.java | 101 +++++++-- .../LibvirtStartNBDServerCommandWrapper.java | 32 +-- .../backup/IncrementalBackupServiceImpl.java | 214 ++++++++++-------- 4 files changed, 231 insertions(+), 132 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index d4ef6652b1ef..b43c46618435 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -25,21 +25,25 @@ public class StartBackupCommand extends Command { private String vmName; private String toCheckpointId; private String fromCheckpointId; + private Long fromCheckpointCreateTime; private int nbdPort; private Map diskPathUuidMap; private String hostIpAddress; + private boolean stoppedVM; public StartBackupCommand() { } - public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, - int nbdPort, Map diskPathUuidMap, String hostIpAddress) { + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, + int nbdPort, Map diskPathUuidMap, String hostIpAddress, boolean stoppedVM) { this.vmName = vmName; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; + this.fromCheckpointCreateTime = fromCheckpointCreateTime; this.nbdPort = nbdPort; this.diskPathUuidMap = diskPathUuidMap; this.hostIpAddress = hostIpAddress; + this.stoppedVM = stoppedVM; } public String getVmName() { @@ -54,6 +58,10 @@ public String getFromCheckpointId() { return fromCheckpointId; } + public Long getFromCheckpointCreateTime() { + return fromCheckpointCreateTime; + } + public int getNbdPort() { return nbdPort; } @@ -70,6 +78,10 @@ public String getHostIpAddress() { return hostIpAddress; } + public boolean isStoppedVM() { + return stoppedVM; + } + @Override public boolean executeInSequence() { return true; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 1dfef22c17e8..bc3faa044933 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -25,13 +25,8 @@ import org.apache.cloudstack.backup.StartBackupCommand; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.libvirt.Connect; -import org.libvirt.Domain; -import org.libvirt.DomainInfo; - import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.StringUtils; @@ -43,22 +38,25 @@ public class LibvirtStartBackupCommandWrapper extends CommandWrapper\n"); + xml.append(" ").append(checkpointName).append("\n"); + xml.append(" ").append(createTime).append("\n"); + xml.append(""); + return xml.toString(); + } + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, int nbdPort, LibvirtComputingResource resource) { StringBuilder xml = new StringBuilder(); xml.append("\n"); @@ -145,4 +184,30 @@ private String createCheckpointXml(String checkpointId) { " " + checkpointId + "\n" + ""; } + + private Answer handleStoppedVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource, String toCheckpointId) { + String vmName = cmd.getVmName(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("sudo"); + script.add("qemu-img"); + script.add("bitmap"); + script.add("--add"); + script.add(diskPath); + script.add(toCheckpointId); + String result = script.execute(); + if (result != null) { + return new StartBackupAnswer(cmd, false, + "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); + } + } + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Stopped VM backup: checkpoint bitmap added successfully", + checkpointCreateTime); + } + + private long getCheckpointCreateTime() { + return System.currentTimeMillis() / 1000; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index c7f2e8d6d08c..7a8588809df6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -32,7 +32,8 @@ public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); - private StartNBDServerAnswer handleUpload(StartNBDServerCommand cmd) { + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { String volumePath = cmd.getVolumePath(); int nbdPort = cmd.getNbdPort(); String hostIpAddress = cmd.getHostIpAddress(); @@ -60,8 +61,14 @@ private StartNBDServerAnswer handleUpload(StartNBDServerCommand cmd) { } String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --bind %s --port %d --persistent %s", - unitName, exportName, hostIpAddress, nbdPort, volumePath + "systemd-run --unit=%s --property=Restart=no " + + "qemu-nbd --export-name %s --bind %s --port %d --persistent %s %s", + unitName, + exportName, + hostIpAddress, + nbdPort, + cmd.getDirection().equals("download") ? "--read-only" : "", + volumePath ); Script startScript = new Script("/bin/bash", logger); @@ -108,23 +115,4 @@ private StartNBDServerAnswer handleUpload(StartNBDServerCommand cmd) { return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for upload", transferId, transferUrl); } - - private StartNBDServerAnswer handleDownload(StartNBDServerCommand cmd) { - String exportName = cmd.getExportName(); - int nbdPort = cmd.getNbdPort(); - String hostIpAddress = cmd.getHostIpAddress(); - String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, exportName); - - return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for download", - cmd.getTransferId(), transferUrl); - } - - @Override - public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { - if (cmd.getDirection().equals("download")) { - return handleDownload(cmd); - } else { - return handleUpload(cmd); - } - } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 618c7667678e..e5894430fbfe 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -140,8 +140,8 @@ public BackupResponse startBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("VM not found: " + vmId); } - if (vm.getState() != State.Running) { - throw new CloudRuntimeException("VM must be running to start backup"); + if (vm.getState() != State.Running && vm.getState() != State.Stopped) { + throw new CloudRuntimeException("VM must be running or stopped to start backup"); } Backup existingBackup = backupDao.findByVmId(vmId); @@ -163,13 +163,15 @@ public BackupResponse startBackup(StartBackupCmd cmd) { String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); String fromCheckpointId = vm.getActiveCheckpointId(); + Long fromCheckpointCreateTime = vm.getActiveCheckpointCreateTime(); backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); int nbdPort = allocateNbdPort(); + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); backup.setNbdPort(nbdPort); - backup.setHostId(vm.getHostId()); + backup.setHostId(hostId); // Will be changed later if incremental was done backup.setType("FULL"); @@ -178,53 +180,52 @@ public BackupResponse startBackup(StartBackupCmd cmd) { List volumes = volumeDao.findByInstance(vmId); Map diskPathUuidMap = new HashMap<>(); for (Volume vol : volumes) { - StoragePoolVO storagePool = primaryDataStoreDao.findById(vol.getPoolId()); - String volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), vol.getPath()); + String volumePath = getVolumePathForFileBasedBackend(vol); diskPathUuidMap.put(volumePath, vol.getUuid()); } - Host host = hostDao.findById(vm.getHostId()); + Host host = hostDao.findById(hostId); StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), toCheckpointId, fromCheckpointId, + fromCheckpointCreateTime, nbdPort, diskPathUuidMap, - host.getPrivateIpAddress() + host.getPrivateIpAddress(), + vm.getState() == State.Stopped ); + StartBackupAnswer answer; try { - StartBackupAnswer answer; - if (dummyOffering) { answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis()); } else { - answer = (StartBackupAnswer) agentManager.send(vm.getHostId(), startCmd); - } - - if (!answer.getResult()) { - backupDao.remove(backup.getId()); - throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); + answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); } - - // Update backup with checkpoint creation time - backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); - if (Boolean.TRUE.equals(answer.getIncremental())) { - // todo: set it in the backend - backup.setType("Incremental"); - } - backupDao.update(backup.getId(), backup); - - BackupResponse response = new BackupResponse(); - response.setId(backup.getUuid()); - response.setVmId(vm.getUuid()); - response.setStatus(backup.getStatus()); - return response; - } catch (AgentUnavailableException | OperationTimedoutException e) { backupDao.remove(backup.getId()); throw new CloudRuntimeException("Failed to communicate with agent", e); } + + if (!answer.getResult()) { + backupDao.remove(backup.getId()); + throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); + } + + // Update backup with checkpoint creation time + backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); + if (Boolean.TRUE.equals(answer.getIncremental())) { + // todo: set it in the backend + backup.setType("Incremental"); + } + backupDao.update(backup.getId(), backup); + + BackupResponse response = new BackupResponse(); + response.setId(backup.getUuid()); + response.setVmId(vm.getUuid()); + response.setStatus(backup.getStatus()); + return response; } @Override @@ -254,40 +255,43 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId); } - StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); + if (vm.getState() == State.Running) { + StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); - try { StopBackupAnswer answer; - if (dummyOffering) { - answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); - } else { - answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); + try { + if (dummyOffering) { + answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); + } else { + answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); + } + + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { throw new CloudRuntimeException("Failed to stop backup: " + answer.getDetails()); } + } - // Update VM checkpoint tracking - String oldCheckpointId = vm.getActiveCheckpointId(); - vm.setActiveCheckpointId(backup.getToCheckpointId()); - vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); - vmInstanceDao.update(vmId, vm); + // Update VM checkpoint tracking + String oldCheckpointId = vm.getActiveCheckpointId(); + vm.setActiveCheckpointId(backup.getToCheckpointId()); + vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); + vmInstanceDao.update(vmId, vm); - // Delete old checkpoint if exists (POC: skip actual libvirt call) - if (oldCheckpointId != null) { - // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete - logger.debug("Would delete old checkpoint: " + oldCheckpointId); - } + // Delete old checkpoint if exists (POC: skip actual libvirt call) + if (oldCheckpointId != null) { + // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete + logger.debug("Would delete old checkpoint: " + oldCheckpointId); + } - // Delete backup session record - backupDao.remove(backup.getId()); + // Delete backup session record + backupDao.remove(backup.getId()); - return true; + return true; - } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent", e); - } } private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume) { @@ -300,6 +304,13 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu String transferId = UUID.randomUUID().toString(); Host host = hostDao.findById(backup.getHostId()); + + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); + if (vm.getState() == State.Stopped) { + String volumePath = getVolumePathForFileBasedBackend(volume); + startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, backup.getNbdPort()); + } + CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( transferId, host.getPrivateIpAddress(), @@ -357,27 +368,16 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { return hosts.get(0); } - private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { - final String direction = ImageTransfer.Direction.upload.toString(); - String transferId = UUID.randomUUID().toString(); - - int nbdPort = allocateNbdPort(); - Long poolId = volume.getPoolId(); - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); - Host host = getFirstHostFromStoragePool(storagePoolVO); - - // todo: This only works with file based storage (not ceph, linbit) - String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); + private void startNBDServer(String transferId, String direction, Host host, String exportName, String volumePath, int nbdPort) { StartNBDServerAnswer nbdServerAnswer; StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, host.getPrivateIpAddress(), - volume.getUuid(), + exportName, volumePath, nbdPort, direction ); - try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(host.getId(), nbdServerCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { @@ -386,6 +386,40 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { if (!nbdServerAnswer.getResult()) { throw new CloudRuntimeException("Failed to start the NBD server"); } + } + + private String getVolumePathForFileBasedBackend(Volume volume) { + Long poolId = volume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + // todo: This only works with file based storage (not ceph, linbit) + String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); + return volumePath; + } + + private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { + final String direction = ImageTransfer.Direction.upload.toString(); + String transferId = UUID.randomUUID().toString(); + int nbdPort = allocateNbdPort(); + + Long poolId = volume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + Host host = getFirstHostFromStoragePool(storagePoolVO); + String volumePath = getVolumePathForFileBasedBackend(volume); + + startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort); + + ImageTransferVO imageTransfer = new ImageTransferVO( + transferId, + null, + volume.getId(), + host.getId(), + nbdPort, + ImageTransferVO.Phase.transferring, + ImageTransfer.Direction.upload, + volume.getAccountId(), + volume.getDomainId(), + volume.getDataCenterId() + ); CreateImageTransferAnswer transferAnswer; CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( @@ -401,22 +435,10 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); if (!transferAnswer.getResult()) { - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); + stopNbdServer(imageTransfer); throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); } - ImageTransferVO imageTransfer = new ImageTransferVO( - transferId, - null, - volume.getId(), - host.getId(), - nbdPort, - ImageTransferVO.Phase.transferring, - ImageTransfer.Direction.upload, - volume.getAccountId(), - volume.getDomainId(), - volume.getDataCenterId() - ); imageTransfer.setTransferUrl(transferAnswer.getTransferUrl()); imageTransfer.setSignedTicketId(transferAnswer.getImageTransferId()); @@ -484,27 +506,44 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } + + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); + if (vm.getState() == State.Stopped) { + boolean stopNbdServerResult = stopNbdServer(imageTransfer); + if (!stopNbdServerResult) { + throw new CloudRuntimeException("Failed to stop the nbd server"); + } + } } - private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { + private boolean stopNbdServer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); int nbdPort = imageTransfer.getNbdPort(); String direction = imageTransfer.getDirection().toString(); - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent", e); + logger.error("Failed to stop NBD server on image transfer finalization", e); + return false; } - if (!answer.getResult()) { + return answer.getResult(); + } + + private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { + String transferId = imageTransfer.getUuid(); + int nbdPort = imageTransfer.getNbdPort(); + String direction = imageTransfer.getDirection().toString(); + + boolean stopNbdServerResult = stopNbdServer(imageTransfer); + if (!stopNbdServerResult) { throw new CloudRuntimeException("Failed to stop the nbd server"); } FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); EndPoint ssvm = _epSelector.findSsvm(imageTransfer.getDataCenterId()); - answer = ssvm.sendMessage(finalizeCmd); + Answer answer = ssvm.sendMessage(finalizeCmd); if (!answer.getResult()) { throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); @@ -527,7 +566,6 @@ public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { } imageTransfer.setPhase(ImageTransferVO.Phase.finished); imageTransferDao.update(imageTransfer.getId(), imageTransfer); - imageTransferDao.remove(imageTransfer.getId()); return true; } @@ -688,15 +726,11 @@ private void pollImageTransferProgress() { String transferId = transfer.getUuid(); transferIds.add(transferId); - String volumePath = volume.getPath(); - if (volumePath == null) { + if (volume.getPath() == null) { logger.warn("Volume path is null for image transfer: " + transfer.getUuid()); continue; } - - StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); - volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), volumePath); - + String volumePath = getVolumePathForFileBasedBackend(volume); volumePaths.put(transferId, volumePath); volumeSizes.put(transferId, volume.getSize()); } From ca4112e7d0ef3ca377b8de1652dc3cb133f7b352 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 27 Jan 2026 08:32:30 +0100 Subject: [PATCH 024/129] api/server: create dummy KVM VM without volume and network is optional --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/command/user/vm/DeployVMCmd.java | 10 ++- .../cloud/vm/VirtualMachineManagerImpl.java | 8 ++- .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 8 +++ .../main/java/com/cloud/vm/UserVmManager.java | 7 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 64 ++++++++++++++++--- 7 files changed, 86 insertions(+), 14 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 05c6098bc726..2e686560a015 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -216,6 +216,7 @@ public class ApiConstants { public static final String DOMAIN_PATH = "domainpath"; public static final String DOMAIN_ID = "domainid"; public static final String DOMAIN__ID = "domainId"; + public static final String DUMMY = "dummy"; public static final String DURATION = "duration"; public static final String ELIGIBLE = "eligible"; public static final String EMAIL = "email"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 050592b97a3b..dd6281ba65e6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -64,6 +64,10 @@ public class DeployVMCmd extends BaseDeployVMCmd { @Parameter(name = ApiConstants.SNAPSHOT_ID, type = CommandType.UUID, entityType = SnapshotResponse.class, since = "4.21") private Long snapshotId; + @Parameter(name = ApiConstants.DUMMY, type = CommandType.BOOLEAN, since = "4.23", description = "Deploy a dummy VM without any disk. False by default. This supports KVM only.") + private Boolean dummy; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -84,6 +88,10 @@ public Long getSnapshotId() { return snapshotId; } + public boolean getDummy() { + return dummy != null && dummy; + } + public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } @@ -132,7 +140,7 @@ public void execute() { @Override public void create() throws ResourceAllocationException { - if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { + if (!getDummy() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID"); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index b20c06fc2c31..423aaececd6c 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -577,7 +577,13 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + if (_userVmMgr.isDummyTemplate(hyperType, template.getId())) { + logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); + return; + } else { + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + } + // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk. CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 4c9f906b68a9..aec06d6d0003 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -106,4 +106,6 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, CPU.CPUArch arch, String urlPathSuffix); + + VMTemplateVO findByAccountAndName(Long accountId, String templateName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9b5d0edc599d..8c6e3fe0983f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -945,4 +945,12 @@ public boolean updateState( } return rows > 0; } + + @Override + public VMTemplateVO findByAccountAndName(Long accountId, String templateName) { + SearchCriteria sc = NameAccountIdSearch.create(); + sc.setParameters("name", templateName); + sc.setParameters("accountId", accountId); + return findOneBy(sc); + } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 0a744709644c..a72498c13718 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -16,13 +16,14 @@ // under the License. package com.cloud.vm; +import static com.cloud.user.ResourceLimitService.ResourceLimitHostTags; + import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import com.cloud.utils.StringUtils; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.framework.config.ConfigKey; @@ -40,8 +41,7 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; - -import static com.cloud.user.ResourceLimitService.ResourceLimitHostTags; +import com.cloud.utils.StringUtils; /** * @@ -204,4 +204,5 @@ static Set getStrictHostTags() { */ boolean isVMPartOfAnyCKSCluster(VMInstanceVO vm); + boolean isDummyTemplate(HypervisorType hypervisorType, Long templateId); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9134be3d3bd9..02a16e7a9e45 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -60,9 +60,6 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.serializer.GsonHelper; -import com.cloud.storage.SnapshotPolicyVO; -import com.cloud.storage.dao.SnapshotPolicyDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -315,6 +312,7 @@ import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; import com.cloud.resourcelimit.CheckedReservation; +import com.cloud.serializer.GsonHelper; import com.cloud.server.ManagementService; import com.cloud.server.ResourceTag; import com.cloud.service.ServiceOfferingVO; @@ -324,8 +322,10 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOSCategoryVO; import com.cloud.storage.GuestOSVO; +import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; @@ -343,7 +343,9 @@ import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; @@ -421,6 +423,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private static final long GiB_TO_BYTES = 1024 * 1024 * 1024; + public static final String KVM_VM_DUMMY_TEMPLATE_NAME = "kvm-vm-dummy-template"; + + @Inject private EntityManager _entityMgr; @Inject @@ -617,6 +622,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject BackupScheduleDao backupScheduleDao; @Inject + LaunchPermissionDao launchPermissionDao; + @Inject private UserDataDao userDataDao; @Inject protected SnapshotHelper snapshotHelper; @@ -651,6 +658,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private boolean _instanceNameFlag; private int _scaleRetry; private Map vmIdCountMap = new ConcurrentHashMap<>(); + private static VMTemplateVO KVM_VM_DUMMY_TEMPLATE; protected static long ROOT_DEVICE_ID = 0; @@ -2498,6 +2506,16 @@ public boolean configure(String name, Map params) throws Configu _vmIpFetchThreadExecutor = Executors.newFixedThreadPool(VmIpFetchThreadPoolMax.value(), new NamedThreadFactory("vmIpFetchThread")); + KVM_VM_DUMMY_TEMPLATE = _templateDao.findByAccountAndName(Account.ACCOUNT_ID_SYSTEM, KVM_VM_DUMMY_TEMPLATE_NAME); + if (KVM_VM_DUMMY_TEMPLATE == null) { + KVM_VM_DUMMY_TEMPLATE = VMTemplateVO.createSystemIso(_templateDao.getNextInSequence(Long.class, "id"), KVM_VM_DUMMY_TEMPLATE_NAME, KVM_VM_DUMMY_TEMPLATE_NAME, true, + "", true, 64, Account.ACCOUNT_ID_SYSTEM, "", + "Dummy Template for KVM VM", false, 1); + KVM_VM_DUMMY_TEMPLATE.setState(VirtualMachineTemplate.State.Active); + KVM_VM_DUMMY_TEMPLATE.setFormat(ImageFormat.QCOW2); + KVM_VM_DUMMY_TEMPLATE = _templateDao.persist(KVM_VM_DUMMY_TEMPLATE); + } + logger.info("User VM Manager is configured."); return true; @@ -3927,7 +3945,9 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service _accountMgr.checkAccess(owner, _diskOfferingDao.findById(diskOfferingId), zone); // If no network is specified, find system security group enabled network - if (networkIdList == null || networkIdList.isEmpty()) { + if (isDummyTemplate(hypervisor, template.getId())) { + logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced security group enabled zone", hypervisor); + } else if (networkIdList == null || networkIdList.isEmpty()) { Network networkWithSecurityGroup = _networkModel.getNetworkWithSGWithFreeIPs(owner, zone.getId()); if (networkWithSecurityGroup == null) { throw new InvalidParameterValueException("No network with security enabled is found in zone id=" + zone.getUuid()); @@ -4040,7 +4060,9 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv _accountMgr.checkAccess(owner, diskOffering, zone); List vpcSupportedHTypes = _vpcMgr.getSupportedVpcHypervisors(); - if (networkIdList == null || networkIdList.isEmpty()) { + if (isDummyTemplate(hypervisor, template.getId())) { + logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced zone", hypervisor); + } else if (networkIdList == null || networkIdList.isEmpty()) { NetworkVO defaultNetwork = getDefaultNetwork(zone, owner, false); if (defaultNetwork != null) { networkList.add(defaultNetwork); @@ -4475,7 +4497,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri } } - if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType)) { + if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType) && !isDummyTemplate(hypervisorType, template.getId())) { throw new InvalidParameterValueException(String.format("Unable to use system template %s to deploy a user vm", template)); } @@ -4488,7 +4510,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri if (CollectionUtils.isEmpty(snapshotsOnZone)) { throw new InvalidParameterValueException("The snapshot does not exist on zone " + zone.getId()); } - } else { + } else if (!isDummyTemplate(hypervisorType, template.getId())) { List listZoneTemplate = _templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { throw new InvalidParameterValueException("The template " + template.getId() + " is not available for use"); @@ -4603,7 +4625,11 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri // by Agent Manager in order to configure default // gateway for the vm if (defaultNetworkNumber == 0) { - throw new InvalidParameterValueException("At least 1 default network has to be specified for the vm"); + if (isDummyTemplate(hypervisorType, template.getId())) { + logger.debug("Template is a dummy template for hypervisor {}, vm can be created without a default network", hypervisorType); + } else { + throw new InvalidParameterValueException("At least 1 default network has to be specified for the vm"); + } } else if (defaultNetworkNumber > 1) { throw new InvalidParameterValueException("Only 1 default network per vm is supported"); } @@ -5321,7 +5347,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "deploying Vm", async = true) public UserVm startVirtualMachine(DeployVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { long vmId = cmd.getEntityId(); - if (!cmd.getStartVm()) { + if (!cmd.getStartVm() || cmd.getDummy()) { return getUserVm(vmId); } Long podId = null; @@ -6469,6 +6495,12 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE (!(HypervisorType.KVM.equals(template.getHypervisorType()) || HypervisorType.KVM.equals(cmd.getHypervisor())))) { throw new InvalidParameterValueException("Deploying a virtual machine with existing volume/snapshot is supported only from KVM hypervisors"); } + if (template == null && HypervisorType.KVM.equals(cmd.getHypervisor()) && cmd.getDummy()) { + template = KVM_VM_DUMMY_TEMPLATE; + logger.info("Creating launch permission for Dummy template"); + LaunchPermissionVO launchPermission = new LaunchPermissionVO(KVM_VM_DUMMY_TEMPLATE.getId(), owner.getId()); + launchPermissionDao.persist(launchPermission); + } // Make sure a valid template ID was specified if (template == null) { throw new InvalidParameterValueException("Unable to use template " + templateId); @@ -6627,6 +6659,12 @@ private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Accoun if (isLeaseFeatureEnabled) { applyLeaseOnCreateInstance(vm, cmd.getLeaseDuration(), cmd.getLeaseExpiryAction(), svcOffering); } + + if (KVM_VM_DUMMY_TEMPLATE != null && template.getId() == KVM_VM_DUMMY_TEMPLATE.getId() && cmd instanceof DeployVMCmd && ((DeployVMCmd) cmd).getDummy()) { + logger.info("Revoking launch permission for Dummy template"); + launchPermissionDao.removePermissions(KVM_VM_DUMMY_TEMPLATE.getId(), Collections.singletonList(owner.getId())); + } + return vm; } @@ -10061,4 +10099,12 @@ private void setVncPasswordForKvmIfAvailable(Map customParameter vm.setVncPassword(customParameters.get(VmDetailConstants.KVM_VNC_PASSWORD)); } } + + @Override + public boolean isDummyTemplate(HypervisorType hypervisorType, Long templateId) { + if (HypervisorType.KVM.equals(hypervisorType) && KVM_VM_DUMMY_TEMPLATE != null && KVM_VM_DUMMY_TEMPLATE.getId() == templateId) { + return true; + } + return false; + } } From a3669298afcc9c3654b032721ea2abca8502209c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 6 Feb 2026 14:05:13 +0530 Subject: [PATCH 025/129] worker vm deployment wip Signed-off-by: Abhishek Kumar --- .../command/admin/vm/DeployVMCmdByAdmin.java | 8 + .../offering/ListServiceOfferingsCmd.java | 12 + .../api/command/user/vm/AddNicToVMCmd.java | 20 + .../api/command/user/vm/BaseDeployVMCmd.java | 212 +++++ .../api/command/user/vm/DeployVMCmd.java | 38 +- .../backup/IncrementalBackupService.java | 4 + .../cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../veeam/adapter/ServerAdapter.java | 833 ++++++++++++++++++ .../veeam/adapter/UserResourceAdapter.java | 345 -------- .../cloudstack/veeam/api/ApiService.java | 4 +- .../veeam/api/ClustersRouteHandler.java | 39 +- .../veeam/api/DataCentersRouteHandler.java | 90 +- .../veeam/api/DisksRouteHandler.java | 50 +- .../veeam/api/HostsRouteHandler.java | 27 +- .../veeam/api/ImageTransfersRouteHandler.java | 37 +- .../veeam/api/JobsRouteHandler.java | 102 +++ .../veeam/api/NetworksRouteHandler.java | 39 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 223 +++-- .../veeam/api/VnicProfilesRouteHandler.java | 39 +- .../AsyncJobJoinVOToJobConverter.java | 50 ++ .../api/converter/NicVOToNicConverter.java | 94 ++ .../converter/UserVmJoinVOToVmConverter.java | 55 +- .../VolumeJoinVOToDiskConverter.java | 20 +- .../veeam/api/dto/DiskAttachment.java | 2 +- .../apache/cloudstack/veeam/api/dto/Ip.java | 61 ++ .../apache/cloudstack/veeam/api/dto/Ips.java | 42 + .../apache/cloudstack/veeam/api/dto/Job.java | 75 ++ .../apache/cloudstack/veeam/api/dto/Jobs.java | 42 + .../VmEntityResponse.java => dto/Mac.java} | 22 +- .../apache/cloudstack/veeam/api/dto/Nic.java | 131 +++ .../apache/cloudstack/veeam/api/dto/Nics.java | 40 + .../veeam/api/dto/ReportedDevice.java | 93 ++ .../veeam/api/dto/ReportedDevices.java | 42 + .../apache/cloudstack/veeam/api/dto/Vm.java | 38 + .../cloudstack/veeam/api/dto/VmAction.java | 51 ++ .../veeam/api/dto/VmInitialization.java | 34 + .../Vms.java} | 10 +- .../apache/cloudstack/veeam/utils/Mapper.java | 2 + .../spring-veeam-control-service-context.xml | 3 +- .../main/java/com/cloud/vm/UserVmManager.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 61 +- .../backup/IncrementalBackupServiceImpl.java | 20 +- 42 files changed, 2432 insertions(+), 682 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ip.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/{response/VmEntityResponse.java => dto/Mac.java} (71%) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/{response/VmCollectionResponse.java => dto/Vms.java} (86%) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java index e64c8b3f46c6..5760bd25a366 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java @@ -48,4 +48,12 @@ public Long getPodId() { public Long getClusterId() { return clusterId; } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 5c5c8776bce3..164a97891bc8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -193,6 +193,18 @@ public Boolean getGpuEnabled() { return gpuEnabled; } + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setCpuNumber(Integer cpuNumber) { + this.cpuNumber = cpuNumber; + } + + public void setMemory(Integer memory) { + this.memory = memory; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java index 6347c38811e8..f6ef955956f6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java @@ -100,6 +100,26 @@ public String getMacAddress() { return NetUtils.standardizeMacAddress(macaddr); } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setNetworkId(Long netId) { + this.netId = netId; + } + + public void setIpaddr(String ipaddr) { + this.ipaddr = ipaddr; + } + + public void setMacAddress(String macaddr) { + this.macaddr = macaddr; + } + + public void setDhcpOptions(Map dhcpOptions) { + this.dhcpOptions = dhcpOptions; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8c29d7338b85..8d02dfa0a793 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -798,6 +798,218 @@ public IoDriverPolicy getIoDriverPolicy() { } return null; } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setName(String name) { + this.name = name; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setPassword(String password) { + this.password = password; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setNetworkIds(List networkIds) { + this.networkIds = networkIds; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public void setBootIntoSetup(Boolean bootIntoSetup) { + this.bootIntoSetup = bootIntoSetup; + } + + public void setDiskOfferingId(Long diskOfferingId) { + this.diskOfferingId = diskOfferingId; + } + + public void setSize(Long size) { + this.size = size; + } + + public void setRootdisksize(Long rootdisksize) { + this.rootdisksize = rootdisksize; + } + + public void setDataDisksDetails(Map dataDisksDetails) { + this.dataDisksDetails = dataDisksDetails; + } + + public void setGroup(String group) { + this.group = group; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public void setUserData(String userData) { + this.userData = userData; + } + + public void setUserdataId(Long userdataId) { + this.userdataId = userdataId; + } + + public void setUserdataDetails(Map userdataDetails) { + this.userdataDetails = userdataDetails; + } + + public void setSshKeyPairName(String sshKeyPairName) { + this.sshKeyPairName = sshKeyPairName; + } + + public void setSshKeyPairNames(List sshKeyPairNames) { + this.sshKeyPairNames = sshKeyPairNames; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } + + public void setSecurityGroupIdList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + + public void setSecurityGroupNameList(List securityGroupNameList) { + this.securityGroupNameList = securityGroupNameList; + } + + public void setIpToNetworkList(Map ipToNetworkList) { + this.ipToNetworkList = ipToNetworkList; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public void setIp6Address(String ip6Address) { + this.ip6Address = ip6Address; + } + + public void setMacAddress(String macAddress) { + this.macAddress = macAddress; + } + + public void setKeyboard(String keyboard) { + this.keyboard = keyboard; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setStartVm(Boolean startVm) { + this.startVm = startVm; + } + + public void setAffinityGroupIdList(List affinityGroupIdList) { + this.affinityGroupIdList = affinityGroupIdList; + } + + public void setAffinityGroupNameList(List affinityGroupNameList) { + this.affinityGroupNameList = affinityGroupNameList; + } + + public void setDisplayVm(Boolean displayVm) { + this.displayVm = displayVm; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void setDeploymentPlanner(String deploymentPlanner) { + this.deploymentPlanner = deploymentPlanner; + } + + public void setDhcpOptionsNetworkList(Map dhcpOptionsNetworkList) { + this.dhcpOptionsNetworkList = dhcpOptionsNetworkList; + } + + public void setDataDiskTemplateToDiskOfferingList(Map dataDiskTemplateToDiskOfferingList) { + this.dataDiskTemplateToDiskOfferingList = dataDiskTemplateToDiskOfferingList; + } + + public void setExtraConfig(String extraConfig) { + this.extraConfig = extraConfig; + } + + public void setCopyImageTags(Boolean copyImageTags) { + this.copyImageTags = copyImageTags; + } + + public void setvAppProperties(Map vAppProperties) { + this.vAppProperties = vAppProperties; + } + + public void setvAppNetworks(Map vAppNetworks) { + this.vAppNetworks = vAppNetworks; + } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } + + public void setOverrideDiskOfferingId(Long overrideDiskOfferingId) { + this.overrideDiskOfferingId = overrideDiskOfferingId; + } + + public void setIothreadsEnabled(Boolean iothreadsEnabled) { + this.iothreadsEnabled = iothreadsEnabled; + } + + public void setIoDriverPolicy(String ioDriverPolicy) { + this.ioDriverPolicy = ioDriverPolicy; + } + + public void setNicMultiqueueNumber(Integer nicMultiqueueNumber) { + this.nicMultiqueueNumber = nicMultiqueueNumber; + } + + public void setNicPackedVirtQueues(Boolean nicPackedVirtQueues) { + this.nicPackedVirtQueues = nicPackedVirtQueues; + } + + public void setLeaseDuration(Integer leaseDuration) { + this.leaseDuration = leaseDuration; + } + + public void setLeaseExpiryAction(String leaseExpiryAction) { + this.leaseExpiryAction = leaseExpiryAction; + } + + public void setExternalDetails(Map externalDetails) { + this.externalDetails = externalDetails; + } + + public void setDataDiskInfoList(List dataDiskInfoList) { + this.dataDiskInfoList = dataDiskInfoList; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index dd6281ba65e6..06b4f64b8592 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -64,8 +64,8 @@ public class DeployVMCmd extends BaseDeployVMCmd { @Parameter(name = ApiConstants.SNAPSHOT_ID, type = CommandType.UUID, entityType = SnapshotResponse.class, since = "4.21") private Long snapshotId; - @Parameter(name = ApiConstants.DUMMY, type = CommandType.BOOLEAN, since = "4.23", description = "Deploy a dummy VM without any disk. False by default. This supports KVM only.") - private Boolean dummy; + @Parameter(name = "blank", type = CommandType.BOOLEAN, since = "4.22.1") + private Boolean blankInstance; ///////////////////////////////////////////////////// @@ -88,14 +88,38 @@ public Long getSnapshotId() { return snapshotId; } - public boolean getDummy() { - return dummy != null && dummy; - } - public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } + public boolean isBlankInstance() { + return Boolean.TRUE.equals(blankInstance); + } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + + public void setServiceOfferingId(Long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public void setBlankInstance(boolean blankInstance) { + this.blankInstance = blankInstance; + } + @Override public void execute() { UserVm result; @@ -140,7 +164,7 @@ public void execute() { @Override public void create() throws ResourceAllocationException { - if (!getDummy() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { + if (!isBlankInstance() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID"); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index 45f73a08dcf7..c37aa5b89eec 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -64,12 +64,16 @@ public interface IncrementalBackupService extends Configurable, PluggableService ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction); + boolean cancelImageTransfer(long imageTransferId); + /** * Finalize an image transfer * Marks transfer as complete (NBD is closed globally in finalize backup) */ boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd); + boolean finalizeImageTransfer(long imageTransferId); + /** * List image transfers for a backup */ diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 423aaececd6c..47b8eba172a0 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -577,7 +577,7 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - if (_userVmMgr.isDummyTemplate(hyperType, template.getId())) { + if (_userVmMgr.isBlankInstanceTemplate(template)) { logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); return; } else { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java new file mode 100644 index 000000000000..0cb2b56d0718 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -0,0 +1,833 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.adapter; + +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RolePermissionEntity; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.Rule; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; +import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; +import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; +import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; +import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.vm.StopVMCmd; +import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.backup.ImageTransfer.Direction; +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.veeam.api.converter.AsyncJobJoinVOToJobConverter; +import org.apache.cloudstack.veeam.api.converter.ClusterVOToClusterConverter; +import org.apache.cloudstack.veeam.api.converter.DataCenterJoinVOToDataCenterConverter; +import org.apache.cloudstack.veeam.api.converter.HostJoinVOToHostConverter; +import org.apache.cloudstack.veeam.api.converter.ImageTransferVOToImageTransferConverter; +import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; +import org.apache.cloudstack.veeam.api.converter.NetworkVOToVnicProfileConverter; +import org.apache.cloudstack.veeam.api.converter.NicVOToNicConverter; +import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; +import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; +import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.api.dto.Cluster; +import org.apache.cloudstack.veeam.api.dto.DataCenter; +import org.apache.cloudstack.veeam.api.dto.Disk; +import org.apache.cloudstack.veeam.api.dto.DiskAttachment; +import org.apache.cloudstack.veeam.api.dto.Host; +import org.apache.cloudstack.veeam.api.dto.ImageTransfer; +import org.apache.cloudstack.veeam.api.dto.Job; +import org.apache.cloudstack.veeam.api.dto.Network; +import org.apache.cloudstack.veeam.api.dto.Nic; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.StorageDomain; +import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.cloudstack.veeam.api.dto.VmAction; +import org.apache.cloudstack.veeam.api.dto.VnicProfile; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import com.cloud.api.query.dao.DataCenterJoinDao; +import com.cloud.api.query.dao.HostJoinDao; +import com.cloud.api.query.dao.ImageStoreJoinDao; +import com.cloud.api.query.dao.StoragePoolJoinDao; +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.dao.VolumeJoinDao; +import com.cloud.api.query.vo.DataCenterJoinVO; +import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.api.query.vo.ImageStoreJoinVO; +import com.cloud.api.query.vo.StoragePoolJoinVO; +import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.org.Grouping; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.dao.AccountDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.EnumUtils; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.NicVO; +import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.UserVmDao; + +public class ServerAdapter extends ManagerBase { + private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; + private static final String SERVICE_ACCOUNT_ROLE_NAME = "Veeam Service Role"; + private static final String SERVICE_ACCOUNT_FIRST_NAME = "Veeam"; + private static final String SERVICE_ACCOUNT_LAST_NAME = "Service User"; + private static final List> SERVICE_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList( + QueryAsyncJobResultCmd.class, + ListVMsCmd.class, + DeployVMCmd.class, + StartVMCmd.class, + StopVMCmd.class, + DestroyVMCmd.class, + ListVolumesCmd.class, + CreateVolumeCmd.class, + DeleteVolumeCmd.class, + AttachVolumeCmd.class, + DetachVolumeCmd.class, + ResizeVolumeCmd.class, + ListNetworksCmd.class + ); + + @Inject + RoleService roleService; + + @Inject + AccountService accountService; + + @Inject + AccountDao accountDao; + + @Inject + DataCenterDao dataCenterDao; + + @Inject + DataCenterJoinDao dataCenterJoinDao; + + @Inject + StoragePoolJoinDao storagePoolJoinDao; + + @Inject + ImageStoreJoinDao imageStoreJoinDao; + + @Inject + ClusterDao clusterDao; + + @Inject + HostJoinDao hostJoinDao; + + @Inject + NetworkDao networkDao; + + @Inject + UserVmDao userVmDao; + + @Inject + UserVmJoinDao userVmJoinDao; + + @Inject + VolumeDao volumeDao; + + @Inject + VolumeJoinDao volumeJoinDao; + + @Inject + VolumeDetailsDao volumeDetailsDao; + + @Inject + VolumeApiService volumeApiService; + + @Inject + PrimaryDataStoreDao primaryDataStoreDao; + + @Inject + ImageTransferDao imageTransferDao; + + @Inject + IncrementalBackupService incrementalBackupService; + + @Inject + QueryService queryService; + + @Inject + ServiceOfferingDao serviceOfferingDao; + + @Inject + UserVmService userVmService; + + @Inject + NicDao nicDao; + + private Map jobsMap = new ConcurrentHashMap<>(); + + protected Role createServiceAccountRole() { + Role role = roleService.createRole(SERVICE_ACCOUNT_ROLE_NAME, RoleType.User, + SERVICE_ACCOUNT_ROLE_NAME, false); + for (Class allowedApi : SERVICE_ACCOUNT_ROLE_ALLOWED_APIS) { + final String apiName = BaseCmd.getCommandNameByClass(allowedApi); + roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW, + String.format("Allow %s", apiName)); + } + roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY, + "Deny all"); + logger.debug("Created default role for Veeam service account in projects: {}", role); + return role; + } + + public Role getServiceAccountRole() { + List roles = roleService.findRolesByName(SERVICE_ACCOUNT_ROLE_NAME); + if (CollectionUtils.isNotEmpty(roles)) { + Role role = roles.get(0); + logger.debug("Found default role for Veeam service account in projects: {}", role); + return role; + } + return createServiceAccountRole(); + } + + protected Account createServiceAccount() { + CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); + try { + Role role = getServiceAccountRole(); + UserAccount userAccount = accountService.createUserAccount(SERVICE_ACCOUNT_NAME, + UUID.randomUUID().toString(), SERVICE_ACCOUNT_FIRST_NAME, + SERVICE_ACCOUNT_LAST_NAME, null, null, SERVICE_ACCOUNT_NAME, Account.Type.NORMAL, role.getId(), + 1L, null, null, null, null, User.Source.NATIVE); + Account account = accountService.getAccount(userAccount.getAccountId()); + logger.debug("Created Veeam service account: {}", account); + return account; + } finally { + CallContext.unregister(); + } + } + + protected Account createServiceAccountIfNeeded() { + List accounts = accountDao.findAccountsByName(SERVICE_ACCOUNT_NAME); + for (AccountVO account : accounts) { + if (Account.State.ENABLED.equals(account.getState())) { + logger.debug("Veeam service account found: {}", account); + return account; + } + } + return createServiceAccount(); + } + + @Override + public boolean start() { + createServiceAccountIfNeeded(); + //find public custom disk offering + return true; + } + + public List listAllDataCenters() { + final List clusters = dataCenterJoinDao.listAll(); + return DataCenterJoinVOToDataCenterConverter.toDCList(clusters); + } + + public DataCenter getDataCenter(String uuid) { + final DataCenterJoinVO vo = dataCenterJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + return DataCenterJoinVOToDataCenterConverter.toDataCenter(vo); + } + + public List listStorageDomainsByDcId(final String uuid) { + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); + if (dataCenterVO == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + List storagePoolVOS = storagePoolJoinDao.listAll(); + List storageDomains = StoreVOToStorageDomainConverter.toStorageDomainListFromPools(storagePoolVOS); + List imageStoreJoinVOS = imageStoreJoinDao.listAll(); + storageDomains.addAll(StoreVOToStorageDomainConverter.toStorageDomainListFromStores(imageStoreJoinVOS)); + return storageDomains; + } + + public List listNetworksByDcId(final String uuid) { + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); + if (dataCenterVO == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + List networks = networkDao.listAll(); + return NetworkVOToNetworkConverter.toNetworkList(networks, (dcId) -> dataCenterVO); + } + + public List listAllClusters() { + final List clusters = clusterDao.listAll(); + return ClusterVOToClusterConverter.toClusterList(clusters, this::getZoneById); + } + + public Cluster getCluster(String uuid) { + final ClusterVO vo = clusterDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); + } + return ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); + } + + public List listAllHosts() { + final List hosts = hostJoinDao.listAll(); + return HostJoinVOToHostConverter.toHostList(hosts); + } + + public Host getHost(String uuid) { + final HostJoinVO vo = hostJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + } + return HostJoinVOToHostConverter.toHost(vo); + } + + public List listAllNetworks() { + final List networks = networkDao.listAll(); + return NetworkVOToNetworkConverter.toNetworkList(networks, this::getZoneById); + } + + public Network getNetwork(String uuid) { + final NetworkVO vo = networkDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + } + return NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); + } + + public List listAllVnicProfiles() { + final List networks = networkDao.listAll(); + return NetworkVOToVnicProfileConverter.toVnicProfileList(networks, this::getZoneById); + } + + public VnicProfile getVnicProfile(String uuid) { + final NetworkVO vo = networkDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + } + return NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); + } + + public List listAllUserVms() { + // Todo: add filtering, pagination + List vms = userVmJoinDao.listAll(); + return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById); + } + + public Vm getVm(String uuid) { + UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::listDiskAttachmentsByInstanceId, + this::listNicsByInstance); + } + + public Vm handleCreateVm(Vm request) { + if (request == null) { + throw new InvalidParameterValueException("Request disk data is empty"); + } + String name = request.name; + Long zoneId = null; + Long clusterId = null; + if (request.cluster != null && StringUtils.isNotEmpty(request.cluster.id)) { + ClusterVO clusterVO = clusterDao.findByUuid(request.cluster.id); + if (clusterVO != null) { + zoneId = clusterVO.getDataCenterId(); + clusterId = clusterVO.getId(); + } + } + if (zoneId == null) { + throw new InvalidParameterValueException("Failed to determine datacenter for VM creation request"); + } + Integer cpu = null; + try { + cpu = request.cpu.topology.sockets; + } catch (Exception ignored) {} + if (cpu == null) { + throw new InvalidParameterValueException("CPU topology sockets must be specified"); + } + Long memory = null; + try { + memory = request.memory; + } catch (Exception ignored) {} + if (memory == null) { + throw new InvalidParameterValueException("Memory must be specified"); + } + String userdata = null; + if (request.getInitialization() != null) { + userdata = request.getInitialization().getContentData(); + } + ApiConstants.BootType bootType = ApiConstants.BootType.BIOS; + ApiConstants.BootMode bootMode = ApiConstants.BootMode.LEGACY; + if (request.bios != null && StringUtils.isNotEmpty(request.bios.type) && request.bios.type.contains("secure")) { + bootType = ApiConstants.BootType.UEFI; + bootMode = ApiConstants.BootMode.SECURE; + } + Account serviceAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + return createVm(zoneId, clusterId, name, cpu, memory, userdata, bootType, bootMode); + } finally { + CallContext.unregister(); + } + } + + protected ServiceOffering getServiceOfferingIdForVmCreation(long zoneId, int cpu, long memory) { + ListServiceOfferingsCmd cmd = new ListServiceOfferingsCmd(); + ComponentContext.inject(cmd); + cmd.setZoneId(zoneId); + cmd.setCpuNumber(cpu); + Integer memoryMB = (int)(memory / (1024L * 1024L)); + cmd.setMemory(memoryMB); + ListResponse offerings = queryService.searchForServiceOfferings(cmd); + if (offerings.getResponses().isEmpty()) { + return null; + } + String uuid = offerings.getResponses().get(0).getId(); + return serviceOfferingDao.findByUuid(uuid); + } + + protected Vm createVm(Long zoneId, Long clusterId, String name, int cpu, long memory, String userdata, + ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { + ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zoneId, cpu, memory); + if (serviceOffering == null) { + throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); + } + DeployVMCmdByAdmin cmd = new DeployVMCmdByAdmin(); + ComponentContext.inject(cmd); + cmd.setZoneId(zoneId); + cmd.setClusterId(clusterId); + cmd.setName(name); + cmd.setServiceOfferingId(serviceOffering.getId()); + if (StringUtils.isNotEmpty(userdata)) { + cmd.setUserData(Base64.getEncoder().encodeToString(userdata.getBytes())); + } + if (bootType != null) { + cmd.setBootType(bootType.toString()); + } + if (bootMode != null) { + cmd.setBootMode(bootMode.toString()); + } + // ToDo: handle other. + cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); + cmd.setBlankInstance(true); + try { + UserVm vm = userVmService.createVirtualMachine(cmd); + vm = userVmService.finalizeCreateVirtualMachine(vm.getId()); + UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::listDiskAttachmentsByInstanceId, + this::listNicsByInstance); + } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { + throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); + } + } + + public void deleteVm(String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + try { + userVmService.destroyVm(vo.getId(), true); + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException("Failed to delete VM: " + e.getMessage(), e); + } + } + + public VmAction startVm(String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + try { + userVmService.startVirtualMachine(vo, null); + return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); + } catch (ResourceUnavailableException | OperationTimedoutException | InsufficientCapacityException | CloudRuntimeException e) { + throw new CloudRuntimeException("Failed to start VM: " + e.getMessage(), e); + } + } + + public VmAction stopVm(String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + try { + userVmService.stopVirtualMachine(vo.getId(), true); + return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); + } catch (CloudRuntimeException e) { + throw new CloudRuntimeException("Failed to stop VM: " + e.getMessage(), e); + } + } + + public VmAction shutdownVm(String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + try { + userVmService.stopVirtualMachine(vo.getId(), false); + return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); + } catch (CloudRuntimeException e) { + throw new CloudRuntimeException("Failed to shutdown VM: " + e.getMessage(), e); + } + } + + public List listAllDisks() { + List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM); + return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes); + } + + public Disk getDisk(String uuid) { + VolumeJoinVO vo = volumeJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + return VolumeJoinVOToDiskConverter.toDisk(vo); + } + + protected List listDiskAttachmentsByInstanceId(final long instanceId) { + List kvmVolumes = volumeJoinDao.listByInstanceId(instanceId); + return VolumeJoinVOToDiskConverter.toDiskAttachmentList(kvmVolumes); + } + + public List listDiskAttachmentsByInstanceUuid(final String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + return listDiskAttachmentsByInstanceId(vo.getId()); + } + + public DiskAttachment handleVmAttachDisk(final String vmUuid, final DiskAttachment request) { + UserVmVO vmVo = userVmDao.findByUuid(vmUuid); + if (vmVo == null) { + throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); + } + if (request == null || request.disk == null || StringUtils.isEmpty(request.disk.id)) { + throw new InvalidParameterValueException("Request disk data is empty"); + } + VolumeVO volumeVO = volumeDao.findByUuid(request.disk.id); + if (volumeVO == null) { + throw new InvalidParameterValueException("Disk with ID " + request.disk.id + " not found"); + } + Account serviceAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), 0L, false); + VolumeJoinVO attachedVolumeVO = volumeJoinDao.findById(volume.getId()); + return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO); + } finally { + CallContext.unregister(); + } + } + + public void deleteDisk(String uuid) { + VolumeVO vo = volumeDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + volumeApiService.deleteVolume(vo.getId(), accountService.getSystemAccount()); + } + + public Disk handleCreateDisk(Disk request) { + if (request == null) { + throw new InvalidParameterValueException("Request disk data is empty"); + } + String name = request.name; + if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { + throw new InvalidParameterValueException("Only worker VM disk creation is supported"); + } + if (request.storageDomains == null || CollectionUtils.isEmpty(request.storageDomains.storageDomain) || + request.storageDomains.storageDomain.size() > 1) { + throw new InvalidParameterValueException("Exactly one storage domain must be specified"); + } + Ref domain = request.storageDomains.storageDomain.get(0); + if (domain == null || domain.id == null) { + throw new InvalidParameterValueException("Storage domain ID must be specified"); + } + StoragePoolVO pool = primaryDataStoreDao.findByUuid(domain.id); + if (pool == null) { + throw new InvalidParameterValueException("Storage domain with ID " + domain.id + " not found"); + } + String sizeStr = request.provisionedSize; + if (StringUtils.isBlank(sizeStr)) { + throw new InvalidParameterValueException("Provisioned size must be specified"); + } + long provisionedSizeInGb; + try { + provisionedSizeInGb = Long.parseLong(sizeStr); + } catch (NumberFormatException ex) { + throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); + } + if (provisionedSizeInGb <= 0) { + throw new InvalidParameterValueException("Provisioned size must be greater than zero"); + } + provisionedSizeInGb = Math.max(1L, provisionedSizeInGb / (1024L * 1024L * 1024L)); + Long initialSize = null; + if (StringUtils.isNotBlank(request.initialSize)) { + try { + initialSize = Long.parseLong(request.initialSize); + } catch (NumberFormatException ignored) {} + } + Account serviceAccount = createServiceAccountIfNeeded(); + DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); + if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { + throw new InvalidParameterValueException("Datacenter for the specified storage domain is not found or not active"); + } + Long diskOfferingId = volumeApiService.getCustomDiskOfferingIdForVolumeUpload(serviceAccount, zone); + if (diskOfferingId == null) { + throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); + } + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); + } finally { + CallContext.unregister(); + } + } + + @NotNull + private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { + Volume volume; + try { + volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, + null, name, sizeInGb, null, null, null, null); + } catch (ResourceAllocationException e) { + throw new CloudRuntimeException(e.getMessage(), e); + } + if (volume == null) { + throw new CloudRuntimeException("Failed to create volume"); + } + volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); + if (initialSize != null) { + volumeDetailsDao.addDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE, String.valueOf(initialSize), true); + } + + // Implementation for creating a Disk resource + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId())); + } + + protected List listNicsByInstance(final long instanceId, final String instanceUuid) { + List nics = nicDao.listByVmId(instanceId); + return NicVOToNicConverter.toNicList(nics, instanceUuid, this::getNetworkById); + } + + protected List listNicsByInstance(final UserVmJoinVO vo) { + return listNicsByInstance(vo.getId(), vo.getUuid()); + } + + public List listNicsByInstanceId(final String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + return listNicsByInstance(vo.getId(), vo.getUuid()); + } + + public Nic handleVmAttachNic(final String vmUuid, final Nic request) { + UserVmVO vmVo = userVmDao.findByUuid(vmUuid); + if (vmVo == null) { + throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); + } + if (request == null || request.getVnicProfile() == null || StringUtils.isEmpty(request.getVnicProfile().id)) { + throw new InvalidParameterValueException("Request nic data is empty"); + } + NetworkVO networkVO = networkDao.findByUuid(request.getVnicProfile().id); + if (networkVO == null) { + throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().id+ " not found"); + } + Account serviceAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + AddNicToVMCmd cmd = new AddNicToVMCmd(); + ComponentContext.inject(cmd); + cmd.setVmId(vmVo.getId()); + cmd.setNetworkId(networkVO.getId()); + if (request.getMac() != null && StringUtils.isNotBlank(request.getMac().getAddress())) { + cmd.setMacAddress(request.getMac().getAddress()); + } + userVmService.addNicToVirtualMachine(cmd); + NicVO nic = nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkVO.getId(), vmVo.getId()); + if (nic == null) { + throw new CloudRuntimeException("Failed to attach NIC to VM"); + } + return NicVOToNicConverter.toNic(nic, vmUuid, this::getNetworkById); + } finally { + CallContext.unregister(); + } + } + + public List listAllImageTransfers() { + List imageTransfers = imageTransferDao.listAll(); + return ImageTransferVOToImageTransferConverter.toImageTransferList(imageTransfers, this::getHostById, this::getVolumeById); + } + + public ImageTransfer getImageTransfer(String uuid) { + ImageTransferVO vo = imageTransferDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); + } + return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); + } + + public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { + if (request == null) { + throw new InvalidParameterValueException("Request image transfer data is empty"); + } + if (request.getDisk() == null || StringUtils.isBlank(request.getDisk().id)) { + throw new InvalidParameterValueException("Disk ID must be specified"); + } + VolumeJoinVO volumeVO = volumeJoinDao.findByUuid(request.getDisk().id); + if (volumeVO == null) { + throw new InvalidParameterValueException("Disk with ID " + request.getDisk().id + " not found"); + } + Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); + if (direction == null) { + throw new InvalidParameterValueException("Invalid or missing direction"); + } + return createImageTransfer(null, volumeVO.getId(), direction); + } + + public boolean handleCancelImageTransfer(String uuid) { + ImageTransferVO vo = imageTransferDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); + } + return incrementalBackupService.cancelImageTransfer(vo.getId()); + } + + public boolean handleFinalizeImageTransfer(String uuid) { + ImageTransferVO vo = imageTransferDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); + } + return incrementalBackupService.finalizeImageTransfer(vo.getId()); + } + + private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction) { + Account serviceAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + try { + org.apache.cloudstack.backup.ImageTransfer imageTransfer = + incrementalBackupService.createImageTransfer(volumeId, null, direction); + ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); + return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); + } finally { + CallContext.unregister(); + } + } + + protected DataCenterJoinVO getZoneById(Long zoneId) { + if (zoneId == null) { + return null; + } + return dataCenterJoinDao.findById(zoneId); + } + + private HostJoinVO getHostById(Long hostId) { + if (hostId == null) { + return null; + } + return hostJoinDao.findById(hostId); + } + + private VolumeJoinVO getVolumeById(Long volumeId) { + if (volumeId == null) { + return null; + } + return volumeJoinDao.findById(volumeId); + } + + protected NetworkVO getNetworkById(Long networkId) { + if (networkId == null) { + return null; + } + return networkDao.findById(networkId); + } + + public List listAllJobs() { + return Collections.emptyList(); + } + + public Job getJob(String uuid) { +// final ClusterVO vo = clusterDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); +// } + long startTime = jobsMap.computeIfAbsent(uuid, k -> System.currentTimeMillis()); + long elapsed = System.currentTimeMillis() - startTime; + if (elapsed > 10000L) { + return AsyncJobJoinVOToJobConverter.toJob(uuid, "finished", startTime); + } else { + return AsyncJobJoinVOToJobConverter.toJob(uuid, "started", startTime); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java deleted file mode 100644 index ad1be6af85e9..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/UserResourceAdapter.java +++ /dev/null @@ -1,345 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.adapter; - -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -import javax.inject.Inject; - -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RolePermissionEntity; -import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.acl.Rule; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; -import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; -import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; -import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; -import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.vm.StopVMCmd; -import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; -import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.cloudstack.backup.ImageTransfer.Direction; -import org.apache.cloudstack.backup.ImageTransferVO; -import org.apache.cloudstack.backup.IncrementalBackupService; -import org.apache.cloudstack.backup.dao.ImageTransferDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.veeam.api.converter.ImageTransferVOToImageTransferConverter; -import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; -import org.apache.cloudstack.veeam.api.dto.Disk; -import org.apache.cloudstack.veeam.api.dto.ImageTransfer; -import org.apache.cloudstack.veeam.api.dto.Ref; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; - -import com.cloud.api.query.dao.HostJoinDao; -import com.cloud.api.query.dao.VolumeJoinDao; -import com.cloud.api.query.vo.HostJoinVO; -import com.cloud.api.query.vo.VolumeJoinVO; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.org.Grouping; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeApiService; -import com.cloud.storage.dao.VolumeDetailsDao; -import com.cloud.user.Account; -import com.cloud.user.AccountService; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.UserAccount; -import com.cloud.user.dao.AccountDao; -import com.cloud.utils.EnumUtils; -import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.exception.CloudRuntimeException; - -public class UserResourceAdapter extends ManagerBase { - private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; - private static final String SERVICE_ACCOUNT_ROLE_NAME = "Veeam Service Role"; - private static final String SERVICE_ACCOUNT_FIRST_NAME = "Veeam"; - private static final String SERVICE_ACCOUNT_LAST_NAME = "Service User"; - private static final List> SERVICE_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList( - QueryAsyncJobResultCmd.class, - ListVMsCmd.class, - DeployVMCmd.class, - StartVMCmd.class, - StopVMCmd.class, - DestroyVMCmd.class, - ListVolumesCmd.class, - CreateVolumeCmd.class, - DeleteVolumeCmd.class, - AttachVolumeCmd.class, - DetachVolumeCmd.class, - ResizeVolumeCmd.class, - ListNetworksCmd.class - ); - - @Inject - DataCenterDao dataCenterDao; - - @Inject - RoleService roleService; - - @Inject - AccountService accountService; - - @Inject - AccountDao accountDao; - - @Inject - VolumeJoinDao volumeJoinDao; - - @Inject - VolumeDetailsDao volumeDetailsDao; - - @Inject - VolumeApiService volumeApiService; - - @Inject - PrimaryDataStoreDao primaryDataStoreDao; - - @Inject - ImageTransferDao imageTransferDao; - - @Inject - HostJoinDao hostJoinDao; - - @Inject - IncrementalBackupService incrementalBackupService; - - protected Role createServiceAccountRole() { - Role role = roleService.createRole(SERVICE_ACCOUNT_ROLE_NAME, RoleType.User, - SERVICE_ACCOUNT_ROLE_NAME, false); - for (Class allowedApi : SERVICE_ACCOUNT_ROLE_ALLOWED_APIS) { - final String apiName = BaseCmd.getCommandNameByClass(allowedApi); - roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW, - String.format("Allow %s", apiName)); - } - roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY, - "Deny all"); - logger.debug("Created default role for Veeam service account in projects: {}", role); - return role; - } - - public Role getServiceAccountRole() { - List roles = roleService.findRolesByName(SERVICE_ACCOUNT_ROLE_NAME); - if (CollectionUtils.isNotEmpty(roles)) { - Role role = roles.get(0); - logger.debug("Found default role for Veeam service account in projects: {}", role); - return role; - } - return createServiceAccountRole(); - } - - protected Account createServiceAccount() { - CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); - try { - Role role = getServiceAccountRole(); - UserAccount userAccount = accountService.createUserAccount(SERVICE_ACCOUNT_NAME, - UUID.randomUUID().toString(), SERVICE_ACCOUNT_FIRST_NAME, - SERVICE_ACCOUNT_LAST_NAME, null, null, SERVICE_ACCOUNT_NAME, Account.Type.NORMAL, role.getId(), - 1L, null, null, null, null, User.Source.NATIVE); - Account account = accountService.getAccount(userAccount.getAccountId()); - logger.debug("Created Veeam service account: {}", account); - return account; - } finally { - CallContext.unregister(); - } - } - - protected Account createServiceAccountIfNeeded() { - List accounts = accountDao.findAccountsByName(SERVICE_ACCOUNT_NAME); - for (AccountVO account : accounts) { - if (Account.State.ENABLED.equals(account.getState())) { - logger.debug("Veeam service account found: {}", account); - return account; - } - } - return createServiceAccount(); - } - - @Override - public boolean start() { - createServiceAccountIfNeeded(); - //find public custom disk offering - return true; - } - - public List listAllDisks() { - List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM); - return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes); - } - - public Disk getDisk(String uuid) { - VolumeJoinVO vo = volumeJoinDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); - } - return VolumeJoinVOToDiskConverter.toDisk(vo); - } - - public Disk handleCreateDisk(Disk request) { - if (request == null) { - throw new InvalidParameterValueException("Request disk data is empty"); - } - String name = request.name; - if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { - throw new InvalidParameterValueException("Only worker VM disk creation is supported"); - } - if (request.storageDomains == null || CollectionUtils.isEmpty(request.storageDomains.storageDomain) || - request.storageDomains.storageDomain.size() > 1) { - throw new InvalidParameterValueException("Exactly one storage domain must be specified"); - } - Ref domain = request.storageDomains.storageDomain.get(0); - if (domain == null || domain.id == null) { - throw new InvalidParameterValueException("Storage domain ID must be specified"); - } - StoragePoolVO pool = primaryDataStoreDao.findByUuid(domain.id); - if (pool == null) { - throw new InvalidParameterValueException("Storage domain with ID " + domain.id + " not found"); - } - String sizeStr = request.provisionedSize; - if (StringUtils.isBlank(sizeStr)) { - throw new InvalidParameterValueException("Provisioned size must be specified"); - } - long provisionedSizeInGb; - try { - provisionedSizeInGb = Long.parseLong(sizeStr); - } catch (NumberFormatException ex) { - throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); - } - if (provisionedSizeInGb <= 0) { - throw new InvalidParameterValueException("Provisioned size must be greater than zero"); - } - provisionedSizeInGb = Math.max(1L, provisionedSizeInGb / (1024L * 1024L * 1024L)); - Long initialSize = null; - if (StringUtils.isNotBlank(request.initialSize)) { - try { - initialSize = Long.parseLong(request.initialSize); - } catch (NumberFormatException ignored) {} - } - Account serviceAccount = createServiceAccountIfNeeded(); - DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); - if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { - throw new InvalidParameterValueException("Datacenter for the specified storage domain is not found or not active"); - } - Long diskOfferingId = volumeApiService.getCustomDiskOfferingIdForVolumeUpload(serviceAccount, zone); - if (diskOfferingId == null) { - throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); - } - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); - try { - return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); - } finally { - CallContext.unregister(); - } - } - - @NotNull - private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { - Volume volume; - try { - volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, - null, name, sizeInGb, null, null, null, null); - } catch (ResourceAllocationException e) { - throw new CloudRuntimeException(e.getMessage(), e); - } - if (volume == null) { - throw new CloudRuntimeException("Failed to create volume"); - } - volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); - if (initialSize != null) { - volumeDetailsDao.addDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE, String.valueOf(initialSize), true); - } - - // Implementation for creating a Disk resource - return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId())); - } - - public List listAllImageTransfers() { - List imageTransfers = imageTransferDao.listAll(); - return ImageTransferVOToImageTransferConverter.toImageTransferList(imageTransfers, this::getHostById, this::getVolumeById); - } - - private HostJoinVO getHostById(Long hostId) { - if (hostId == null) { - return null; - } - return hostJoinDao.findById(hostId); - } - - private VolumeJoinVO getVolumeById(Long volumeId) { - if (volumeId == null) { - return null; - } - return volumeJoinDao.findById(volumeId); - } - - public ImageTransfer getImageTransfer(String uuid) { - ImageTransferVO vo = imageTransferDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); - } - return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); - } - - public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { - if (request == null) { - throw new InvalidParameterValueException("Request image transfer data is empty"); - } - if (request.getDisk() == null || StringUtils.isBlank(request.getDisk().id)) { - throw new InvalidParameterValueException("Disk ID must be specified"); - } - VolumeJoinVO volumeVO = volumeJoinDao.findByUuid(request.getDisk().id); - if (volumeVO == null) { - throw new InvalidParameterValueException("Disk with ID " + request.getDisk().id + " not found"); - } - Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); - if (direction == null) { - throw new InvalidParameterValueException("Invalid or missing direction"); - } - return createImageTransfer(null, volumeVO.getId(), direction); - } - - private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction) { - Account serviceAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); - try { - org.apache.cloudstack.backup.ImageTransfer imageTransfer = - incrementalBackupService.createImageTransfer(volumeId, null, direction); - ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); - return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); - } finally { - CallContext.unregister(); - } - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index 24a9dbb730ee..380a64715fea 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -32,13 +32,13 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.api.dto.Api; +import org.apache.cloudstack.veeam.api.dto.ApiSummary; import org.apache.cloudstack.veeam.api.dto.EmptyElement; import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.ProductInfo; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.SpecialObjectRef; import org.apache.cloudstack.veeam.api.dto.SpecialObjects; -import org.apache.cloudstack.veeam.api.dto.ApiSummary; import org.apache.cloudstack.veeam.api.dto.SummaryCount; import org.apache.cloudstack.veeam.api.dto.Version; import org.apache.cloudstack.veeam.utils.Negotiation; @@ -65,7 +65,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } private void handleRootApiRequest(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - io.getWriter().write(resp, 200, + io.getWriter().write(resp, HttpServletResponse.SC_OK, createDummyApi(VeeamControlService.ContextPath.value() + BASE_ROUTE), outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index 4c4dda45f8ce..a80d0ec8d611 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -26,27 +26,21 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.ClusterVOToClusterConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.Clusters; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.DataCenterJoinDao; -import com.cloud.api.query.vo.DataCenterJoinVO; -import com.cloud.dc.ClusterVO; -import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; public class ClustersRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/clusters"; @Inject - ClusterDao clusterDao; - - @Inject - DataCenterJoinDao dataCenterJoinDao; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -90,32 +84,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = ClusterVOToClusterConverter.toClusterList(listClusters(), this::getZoneById); + final List result = serverAdapter.listAllClusters(); final Clusters response = new Clusters(result); - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listClusters() { - return clusterDao.listAll(); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final ClusterVO vo = clusterDao.findByUuid(id); - if (vo == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; - } - Cluster response = ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); - - io.getWriter().write(resp, 200, response, outFormat); - } - - protected DataCenterJoinVO getZoneById(Long zoneId) { - if (zoneId == null) { - return null; + try { + Cluster response = serverAdapter.getCluster(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - return dataCenterJoinDao.findById(zoneId); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 5c84a20bc103..e2e60fe8479e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -26,9 +26,7 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.DataCenterJoinVOToDataCenterConverter; -import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; -import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.DataCenter; import org.apache.cloudstack.veeam.api.dto.DataCenters; import org.apache.cloudstack.veeam.api.dto.Network; @@ -39,33 +37,14 @@ import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.DataCenterJoinDao; -import com.cloud.api.query.dao.ImageStoreJoinDao; -import com.cloud.api.query.dao.StoragePoolJoinDao; -import com.cloud.api.query.vo.DataCenterJoinVO; -import com.cloud.api.query.vo.ImageStoreJoinVO; -import com.cloud.api.query.vo.StoragePoolJoinVO; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; public class DataCentersRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/datacenters"; - private static final int DEFAULT_MAX = 50; - private static final int HARD_CAP_MAX = 1000; - private static final int DEFAULT_PAGE = 1; @Inject - DataCenterJoinDao dataCenterJoinDao; - - @Inject - StoragePoolJoinDao storagePoolJoinDao; - - @Inject - ImageStoreJoinDao imageStoreJoinDao; - - @Inject - NetworkDao networkDao; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -119,66 +98,41 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = DataCenterJoinVOToDataCenterConverter.toDCList(listDCs()); + final List result = serverAdapter.listAllDataCenters(); final DataCenters response = new DataCenters(result); - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listDCs() { - return dataCenterJoinDao.listAll(); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); - if (dataCenterVO == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; + try { + DataCenter response = serverAdapter.getDataCenter(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - DataCenter response = DataCenterJoinVOToDataCenterConverter.toDataCenter(dataCenterVO); - - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listStoragePoolsByDcId(final long dcId) { - return storagePoolJoinDao.listAll(); - } - - protected List listImageStoresByDcId(final long dcId) { - return imageStoreJoinDao.listAll(); - } - - protected List listNetworksByDcId(final long dcId) { - return networkDao.listAll(); } protected void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); - if (dataCenterVO == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; + try { + List storageDomains = serverAdapter.listStorageDomainsByDcId(id); + StorageDomains response = new StorageDomains(storageDomains); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - List storageDomains = StoreVOToStorageDomainConverter.toStorageDomainListFromPools(listStoragePoolsByDcId(dataCenterVO.getId())); - storageDomains.addAll(StoreVOToStorageDomainConverter.toStorageDomainListFromStores(listImageStoresByDcId(dataCenterVO.getId()))); - - StorageDomains response = new StorageDomains(storageDomains); - - io.getWriter().write(resp, 200, response, outFormat); } protected void handleGetNetworksByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(id); - if (dataCenterVO == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; + try { + List networks = serverAdapter.listNetworksByDcId(id); + Networks response = new Networks(networks); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - List networks = NetworkVOToNetworkConverter.toNetworkList(listNetworksByDcId(dataCenterVO.getId()), (dcId) -> dataCenterVO); - - Networks response = new Networks(networks); - - io.getWriter().write(resp, 200, response, outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 6cac244e1335..0bd618a8111e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -26,7 +26,7 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.adapter.UserResourceAdapter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.Disks; import org.apache.cloudstack.veeam.utils.Negotiation; @@ -42,7 +42,7 @@ public class DisksRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/disks"; @Inject - UserResourceAdapter userResourceAdapter; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -74,16 +74,22 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - if (!"GET".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET", outFormat); - return; - } List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); if (CollectionUtils.isNotEmpty(idAndSubPath)) { String id = idAndSubPath.get(0); if (idAndSubPath.size() == 1) { - handleGetById(id, resp, outFormat, io); - return; + if (!"GET".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, DELETE", outFormat); + return; + } + if ("GET".equalsIgnoreCase(method)) { + handleGetById(id, resp, outFormat, io); + return; + } + if ("DELETE".equalsIgnoreCase(method)) { + handleDeleteById(id, resp, outFormat, io); + return; + } } } @@ -92,32 +98,42 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = userResourceAdapter.listAllDisks(); + final List result = serverAdapter.listAllDisks(); final Disks response = new Disks(result); - io.getWriter().write(resp, 200, response, outFormat); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req); - logger.info("Received POST request on /api/disks endpoint, but method: POST is not supported atm. Request-data: {}", data); + logger.info("Received POST request on /api/disks endpoint. Request-data: {}", data); // ToDo: remove try { Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); - Disk response = userResourceAdapter.handleCreateDisk(request); - io.getWriter().write(resp, 201, response, outFormat); + Disk response = serverAdapter.handleCreateDisk(request); + io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().write(resp, 400, e.getMessage(), outFormat); + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - Disk response = userResourceAdapter.getDisk(id); - io.getWriter().write(resp, 200, response, outFormat); + Disk response = serverAdapter.getDisk(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handleDeleteById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + serverAdapter.deleteDisk(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, "Deleted disk ID: " + id, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().write(resp, 404, e.getMessage(), outFormat); + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java index 6ed3a3af0b7d..37ac17b23642 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -26,22 +26,21 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.HostJoinVOToHostConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Host; import org.apache.cloudstack.veeam.api.dto.Hosts; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.HostJoinDao; -import com.cloud.api.query.vo.HostJoinVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; public class HostsRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/hosts"; @Inject - HostJoinDao hostJoinDao; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -85,25 +84,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = HostJoinVOToHostConverter.toHostList(listHosts()); + final List result = serverAdapter.listAllHosts(); final Hosts response = new Hosts(result); - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listHosts() { - return hostJoinDao.listAll(); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final HostJoinVO vo = hostJoinDao.findByUuid(id); - if (vo == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; + try { + Host response = serverAdapter.getHost(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - Host response = HostJoinVOToHostConverter.toHost(vo); - - io.getWriter().write(resp, 200, response, outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index a469afc08b54..3cdd5d0469d8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -26,7 +26,7 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.adapter.UserResourceAdapter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.ImageTransfer; import org.apache.cloudstack.veeam.api.dto.ImageTransfers; import org.apache.cloudstack.veeam.utils.Negotiation; @@ -42,7 +42,7 @@ public class ImageTransfersRouteHandler extends ManagerBase implements RouteHand public static final String BASE_ROUTE = "/api/imagetransfers"; @Inject - UserResourceAdapter userResourceAdapter; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -105,11 +105,10 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = userResourceAdapter.listAllImageTransfers(); + final List result = serverAdapter.listAllImageTransfers(); final ImageTransfers response = new ImageTransfers(); response.setImageTransfer(result); - - io.getWriter().write(resp, 400, response, outFormat); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, @@ -118,32 +117,40 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons logger.info("Received POST request on /api/imagetransfers endpoint. Request-data: {}", data); try { ImageTransfer request = io.getMapper().jsonMapper().readValue(data, ImageTransfer.class); - ImageTransfer response = userResourceAdapter.handleCreateImageTransfer(request); - io.getWriter().write(resp, 201, response, outFormat); + ImageTransfer response = serverAdapter.handleCreateImageTransfer(request); + io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().write(resp, 400, e.getMessage(), outFormat); + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad Request", e.getMessage(), outFormat); } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - ImageTransfer response = userResourceAdapter.getImageTransfer(id); - io.getWriter().write(resp, 200, response, outFormat); + ImageTransfer response = serverAdapter.getImageTransfer(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().write(resp, 404, e.getMessage(), outFormat); + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } } protected void handleCancelById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - //ToDo: implement cancel logic - io.getWriter().write(resp, 200, "Image transfer cancelled successfully", outFormat); + try { + serverAdapter.handleCancelImageTransfer(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer cancelled successfully", outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } } protected void handleFinalizeById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - //ToDo: implement finalize logic - io.getWriter().write(resp, 200, "Image transfer finalized successfully", outFormat); + try { + serverAdapter.handleFinalizeImageTransfer(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer finalized successfully", outFormat); + } catch (CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java new file mode 100644 index 000000000000..516ea8de4d8e --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; +import org.apache.cloudstack.veeam.api.dto.Job; +import org.apache.cloudstack.veeam.api.dto.Jobs; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.component.ManagerBase; + +public class JobsRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/jobs"; + + @Inject + ServerAdapter serverAdapter; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; + } + } + + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + } + + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = serverAdapter.listAllJobs(); + final Jobs response = new Jobs(result); + + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } + + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + Job response = serverAdapter.getJob(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index 2b895a2a647c..d11397e1eee5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -26,27 +26,21 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.Networks; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.DataCenterJoinDao; -import com.cloud.api.query.vo.DataCenterJoinVO; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; public class NetworksRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/networks"; @Inject - NetworkDao networkDao; - - @Inject - DataCenterJoinDao dataCenterJoinDao; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -90,32 +84,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = NetworkVOToNetworkConverter.toNetworkList(listNetworks(), this::getZoneById); + final List result = serverAdapter.listAllNetworks(); final Networks response = new Networks(result); - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listNetworks() { - return networkDao.listAll(); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final NetworkVO vo = networkDao.findByUuid(id); - if (vo == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; - } - Network response = NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); - - io.getWriter().write(resp, 200, response, outFormat); - } - - protected DataCenterJoinVO getZoneById(Long zoneId) { - if (zoneId == null) { - return null; + try { + Network response = serverAdapter.getNetwork(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - return dataCenterJoinDao.findById(zoneId); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 6971c81b69fb..30d781e868b6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -27,27 +27,26 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; -import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.DiskAttachments; +import org.apache.cloudstack.veeam.api.dto.Nic; +import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.cloudstack.veeam.api.dto.VmAction; +import org.apache.cloudstack.veeam.api.dto.Vms; import org.apache.cloudstack.veeam.api.request.VmListQuery; import org.apache.cloudstack.veeam.api.request.VmSearchExpr; import org.apache.cloudstack.veeam.api.request.VmSearchFilters; import org.apache.cloudstack.veeam.api.request.VmSearchParser; -import org.apache.cloudstack.veeam.api.response.VmCollectionResponse; -import org.apache.cloudstack.veeam.api.response.VmEntityResponse; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.HostJoinDao; -import com.cloud.api.query.dao.UserVmJoinDao; -import com.cloud.api.query.dao.VolumeJoinDao; -import com.cloud.api.query.vo.HostJoinVO; -import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.fasterxml.jackson.core.JsonProcessingException; public class VmsRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/vms"; @@ -56,13 +55,7 @@ public class VmsRouteHandler extends ManagerBase implements RouteHandler { private static final int DEFAULT_PAGE = 1; @Inject - UserVmJoinDao userVmJoinDao; - - @Inject - HostJoinDao hostJoinDao; - - @Inject - VolumeJoinDao volumeJoinDao; + ServerAdapter serverAdapter; private VmSearchParser searchParser; @@ -90,24 +83,74 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path final String method = req.getMethod(); final String sanitizedPath = getSanitizedPath(path); if (sanitizedPath.equals(BASE_ROUTE)) { - if (!"GET".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET", outFormat); + if (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, POST, DELETE", outFormat); + return; + } + if ("GET".equalsIgnoreCase(method)) { + handleGet(req, resp, outFormat, io); + return; + } + if ("POST".equalsIgnoreCase(method)) { + handlePost(req, resp, outFormat, io); return; } - handleGet(req, resp, outFormat, io); - return; } List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); if (CollectionUtils.isNotEmpty(idAndSubPath)) { String id = idAndSubPath.get(0); if (idAndSubPath.size() == 1) { - handleGetById(id, resp, outFormat, io); + if (!"GET".equalsIgnoreCase(method) && !"PUT".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, PUT, DELETE", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetById(id, resp, outFormat, io); + } else if ("DELETE".equalsIgnoreCase(method)) { + handleUpdateById(id, req, resp, outFormat, io); + } else if ("DELETE".equalsIgnoreCase(method)) { + handleDeleteById(id, resp, outFormat, io); + } return; } else if (idAndSubPath.size() == 2) { String subPath = idAndSubPath.get(1); - if ("diskattachments".equals(subPath)) { - handleGetDisAttachmentsByVmId(id, resp, outFormat, io); + if ("start".equals(subPath)) { + if ("POST".equalsIgnoreCase(method)) { + handleStartVmById(id, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; + } else if ("stop".equals(subPath)) { + if ("POST".equalsIgnoreCase(method)) { + handleStopVmById(id, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; + } else if ("shutdown".equals(subPath)) { + if ("POST".equalsIgnoreCase(method)) { + handleShutdownVmById(id, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; + } else if ("diskattachments".equals(subPath)) { + if (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, POST", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetDiskAttachmentsByVmId(id, resp, outFormat, io); + } else if ("POST".equalsIgnoreCase(method)) { + handlePostDiskAttachmentForVmId(id, req, resp, outFormat, io); + } + return; + } else if ("nics".equals(subPath)) { + if (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, POST", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetNicsByVmId(id, resp, outFormat, io); + } else if ("POST".equalsIgnoreCase(method)) { + handlePostNicForVmId(id, req, resp, outFormat, io); + } return; } } @@ -149,10 +192,10 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse return; } - final List result = UserVmJoinVOToVmConverter.toVmList(listUserVms(), this::getHostById); - final VmCollectionResponse response = new VmCollectionResponse(result); + final List result = serverAdapter.listAllUserVms(); + final Vms response = new Vms(result); - io.getWriter().write(resp, 200, response, outFormat); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected static VmListQuery fromRequest(final HttpServletRequest req) { @@ -172,41 +215,123 @@ protected static Integer parseIntOrNull(final String s) { } } - protected List listUserVms() { - // Todo: add filtering, pagination - return userVmJoinDao.listAll(); + protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received method: POST request. Request-data: {}", data); + try { + Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); + Vm response = serverAdapter.handleCreateVm(request); + io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); - if (userVmJoinVO == null) { - io.notFound(resp, "VM not found: " + id, outFormat); - return; + try { + Vm response = serverAdapter.getVm(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - VmEntityResponse response = new VmEntityResponse(UserVmJoinVOToVmConverter.toVm(userVmJoinVO, this::getHostById)); + } - io.getWriter().write(resp, 200, response, outFormat); + protected void handleUpdateById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received POST request, but method: POST is not supported atm. Request-data: {}", data); + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Not implemented", "", outFormat); } - protected void handleGetDisAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { - final UserVmJoinVO userVmJoinVO = userVmJoinDao.findByUuid(id); - if (userVmJoinVO == null) { - io.notFound(resp, "VM not found: " + id, outFormat); - return; + protected void handleDeleteById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + serverAdapter.deleteVm(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, "", outFormat); + } catch (CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - List disks = VolumeJoinVOToDiskConverter.toDiskAttachmentList( - volumeJoinDao.listByInstanceId(userVmJoinVO.getId())); - DiskAttachments response = new DiskAttachments(disks); + } - io.getWriter().write(resp, 200, response, outFormat); + protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + VmAction vm = serverAdapter.startVm(id); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); + } catch (CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + VmAction vm = serverAdapter.stopVm(id); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); + } catch (CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + VmAction vm = serverAdapter.shutdownVm(id); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); + } catch (CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + List disks = serverAdapter.listDiskAttachmentsByInstanceUuid(id); + DiskAttachments response = new DiskAttachments(disks); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handlePostDiskAttachmentForVmId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received method: POST request. Request-data: {}", data); + try { + DiskAttachment request = io.getMapper().jsonMapper().readValue(data, DiskAttachment.class); + DiskAttachment response = serverAdapter.handleVmAttachDisk(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + } } - protected HostJoinVO getHostById(Long hostId) { - if (hostId == null) { - return null; + protected void handleGetNicsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + List nics = serverAdapter.listNicsByInstanceId(id); + Nics response = new Nics(nics); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + } + } + + protected void handlePostNicForVmId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = RouteHandler.getRequestData(req); + logger.info("Received method: POST request. Request-data: {}", data); + try { + Nic request = io.getMapper().jsonMapper().readValue(data, Nic.class); + Nic response = serverAdapter.handleVmAttachNic(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); } - return hostJoinDao.findById(hostId); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index ba7e040e4559..c62fbf694821 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -26,27 +26,21 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.api.converter.NetworkVOToVnicProfileConverter; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.VnicProfile; import org.apache.cloudstack.veeam.api.dto.VnicProfiles; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import com.cloud.api.query.dao.DataCenterJoinDao; -import com.cloud.api.query.vo.DataCenterJoinVO; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; public class VnicProfilesRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/vnicprofiles"; @Inject - NetworkDao networkDao; - - @Inject - DataCenterJoinDao dataCenterJoinDao; + ServerAdapter serverAdapter; @Override public boolean start() { @@ -90,32 +84,19 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = NetworkVOToVnicProfileConverter.toVnicProfileList(listNetworks(), this::getZoneById); + final List result = serverAdapter.listAllVnicProfiles(); final VnicProfiles response = new VnicProfiles(result); - io.getWriter().write(resp, 200, response, outFormat); - } - - protected List listNetworks() { - return networkDao.listAll(); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - final NetworkVO vo = networkDao.findByUuid(id); - if (vo == null) { - io.notFound(resp, "DataCenter not found: " + id, outFormat); - return; - } - VnicProfile response = NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); - - io.getWriter().write(resp, 200, response, outFormat); - } - - protected DataCenterJoinVO getZoneById(Long zoneId) { - if (zoneId == null) { - return null; + try { + VnicProfile response = serverAdapter.getVnicProfile(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); } - return dataCenterJoinDao.findById(zoneId); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java new file mode 100644 index 000000000000..f3aa1dd40025 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.Collections; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.JobsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Actions; +import org.apache.cloudstack.veeam.api.dto.Job; +import org.apache.cloudstack.veeam.api.dto.Ref; + +public class AsyncJobJoinVOToJobConverter { + + public static Job toJob(String uuid, String state, long startTime) { + Job job = new Job(); + final String basePath = VeeamControlService.ContextPath.value(); + // Fill in dummy data for now, as the AsyncJobJoinVO does not contain all the necessary information to populate a Job object. + job.setId(uuid); + job.setHref(basePath + JobsRouteHandler.BASE_ROUTE + "/" + uuid); + job.setAutoCleared(Boolean.TRUE.toString()); + job.setExternal(Boolean.TRUE.toString()); + job.setLastUpdated(System.currentTimeMillis()); + job.setStartTime(startTime); + job.setStatus(state); + if ("complete".equalsIgnoreCase(state) || "finished".equalsIgnoreCase(state)) { + job.setEndTime(System.currentTimeMillis()); + } + job.setOwner(Ref.of(basePath + "/api/users/" + uuid, uuid)); + job.setActions(new Actions()); + job.setDescription("Something"); + job.setLink(Collections.emptyList()); + return job; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java new file mode 100644 index 000000000000..72fe2d559656 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.VnicProfilesRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Ip; +import org.apache.cloudstack.veeam.api.dto.Ips; +import org.apache.cloudstack.veeam.api.dto.Mac; +import org.apache.cloudstack.veeam.api.dto.Nic; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.ReportedDevice; +import org.apache.cloudstack.veeam.api.dto.ReportedDevices; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import com.cloud.network.dao.NetworkVO; +import com.cloud.vm.NicVO; + +public class NicVOToNicConverter { + + public static Nic toNic(final NicVO vo, final String vmUuid, final Function networkResolver) { + final String basePath = VeeamControlService.ContextPath.value(); + final Nic nic = new Nic(); + nic.setId(vo.getUuid()); + nic.setName(vo.getReserver()); + Mac mac = new Mac(); + mac.setAddress(vo.getMacAddress()); + nic.setMac(mac); + nic.setLinked(true); + nic.setPlugged(true); + if (StringUtils.isBlank(vmUuid)) { + nic.setVm(Ref.of(basePath + "/vms/" + vmUuid, vmUuid)); + nic.setHref(nic.getVm().href + "/nics/" + vo.getUuid()); + } + nic.setInterfaceType("virtio"); + ReportedDevice device = getReportedDevice(vo, mac, nic.getVm()); + nic.setReportedDevices(new ReportedDevices(List.of(device))); + if (networkResolver != null) { + final NetworkVO network = networkResolver.apply(vo.getNetworkId()); + if (network != null) { + nic.setVnicProfile(Ref.of(basePath + VnicProfilesRouteHandler.BASE_ROUTE + "/" + network.getUuid(), network.getUuid())); + } + } + return nic; + } + + @NotNull + private static ReportedDevice getReportedDevice(NicVO vo, Mac mac, Ref vm) { + ReportedDevice device = new ReportedDevice(); + device.setType("network"); + device.setId(vo.getUuid()); + device.setName("eth0"); + device.setMac(mac); + Ip ip = new Ip(); + if (vo.getIPv4Address() != null) { + ip.setAddress(vo.getIPv4Address()); + ip.setGateway(vo.getIPv4Gateway()); + ip.setVersion("v4"); + } else if (vo.getIPv6Address() != null) { + ip.setAddress(vo.getIPv6Address()); + ip.setGateway(vo.getIPv6Gateway()); + ip.setVersion("v6"); + } + device.setIps(new Ips(List.of(ip))); + device.setVm(vm); + return device; + } + + public static List toNicList(final List vos, final String vmUuid, final Function networkResolver) { + return vos.stream() + .map(vo -> toNic(vo, vmUuid, networkResolver)) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 7216eb89af12..a4f59dfee52d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -25,14 +25,22 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.JobsRouteHandler; import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Bios; import org.apache.cloudstack.veeam.api.dto.Cpu; +import org.apache.cloudstack.veeam.api.dto.DiskAttachment; +import org.apache.cloudstack.veeam.api.dto.DiskAttachments; +import org.apache.cloudstack.veeam.api.dto.EmptyElement; import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.Nic; +import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.Os; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.cloudstack.veeam.api.dto.VmAction; import org.apache.commons.lang3.StringUtils; import com.cloud.api.query.vo.HostJoinVO; @@ -49,7 +57,8 @@ private UserVmJoinVOToVmConverter() { * * @param src UserVmJoinVO */ - public static Vm toVm(final UserVmJoinVO src, final Function hostResolver) { + public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, + final Function> disksResolver, final Function> nicsResolver) { if (src == null) { return null; } @@ -62,10 +71,13 @@ public static Vm toVm(final UserVmJoinVO src, final Function h dst.description = src.getDisplayName(); dst.href = basePath + VmsRouteHandler.BASE_ROUTE + "/" + src.getUuid(); dst.status = mapStatus(src.getState()); - final Date lastUpdated = src.getLastUpdated(); + final Date lastUpdated = src.getLastUpdated() != null ? src.getLastUpdated() : src.getCreated(); if ("down".equals(dst.status)) { dst.stopTime = lastUpdated.getTime(); } + if ("up".equals(dst.status)) { + dst.setStartTime(lastUpdated.getTime()); + } final Ref template = buildRef( basePath + ApiService.BASE_ROUTE, "templates", @@ -93,24 +105,35 @@ public static Vm toVm(final UserVmJoinVO src, final Function h hostVo.getClusterUuid()); } } - Long hostId = src.getHostId() != null ? src.getHostId() : src.getLastHostId(); - if (hostId != null) { - // I want to get Host data from hostJoinDao but this is a static method without dao access. - - } dst.memory = src.getRamSize() * 1024L * 1024L; - dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), src.getCpu(), 1)); + dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); dst.os = new Os(); dst.os.type = src.getGuestOsId() % 2 == 0 ? "windows" : "linux"; dst.bios = new Bios(); dst.bios.type = "q35_secure_boot"; - dst.type = "server"; + dst.type = "desktop"; dst.origin = "ovirt"; - dst.actions = null;dst.link = List.of( + + if (disksResolver != null) { + List diskAttachments = disksResolver.apply(src.getId()); + dst.setDiskAttachments(new DiskAttachments(diskAttachments)); + } + + if (disksResolver != null) { + List nics = nicsResolver.apply(src); + dst.setNics(new Nics(nics)); + } + + dst.actions = new Actions(List.of( + new Link("start", dst.href + "/start"), + new Link("stop", dst.href + "/stop"), + new Link("shutdown", dst.href + "/shutdown") + )); + dst.link = List.of( new Link("diskattachments", dst.href + "/diskattachments"), new Link("nics", @@ -118,16 +141,26 @@ public static Vm toVm(final UserVmJoinVO src, final Function h new Link("snapshots", dst.href + "/snapshots") ); + dst.tags = new EmptyElement(); return dst; } public static List toVmList(final List srcList, final Function hostResolver) { return srcList.stream() - .map(v -> toVm(v, hostResolver)) + .map(v -> toVm(v, hostResolver, null, null)) .collect(Collectors.toList()); } + public static VmAction toVmAction(final UserVmJoinVO vm) { + VmAction action = new VmAction(); + final String basePath = VeeamControlService.ContextPath.value(); + action.setVm(toVm(vm, null, null, null)); + action.setJob(Ref.of(basePath + JobsRouteHandler.BASE_ROUTE + vm.getUuid(), vm.getUuid())); + action.setStatus("complete"); + return action; + } + private static String mapStatus(final VirtualMachine.State state) { if (state == null) { return null; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 44f56bfbd008..0bb8e40d92a3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.DisksRouteHandler; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; @@ -132,23 +133,20 @@ public static List toDiskList(final List srcList) { public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { final DiskAttachment da = new DiskAttachment(); - final String apiBase = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; + final String basePath = VeeamControlService.ContextPath.value(); + final String apiBase = basePath + ApiService.BASE_ROUTE; final String diskAttachmentId = vol.getUuid(); - final String diskAttachmentHref = apiBase + "/diskattachments/" + diskAttachmentId; + da.vm = Ref.of( + basePath + VmsRouteHandler.BASE_ROUTE + "/" + vol.getVmUuid(), + vol.getVmUuid() + ); da.id = diskAttachmentId; - da.href = diskAttachmentHref; + da.href = da.vm.href + "/diskattachements/" + diskAttachmentId;; // Links - da.disk = Ref.of( - apiBase + "/disks/" + vol.getUuid(), - vol.getUuid() - ); - da.vm = Ref.of( - apiBase + "/vms/" + vol.getVmUuid(), - vol.getVmUuid() - ); + da.disk = toDisk(vol); // Properties da.active = "true"; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java index ca041e993f5e..578b9462c41d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java @@ -43,7 +43,7 @@ public final class DiskAttachment { @JsonProperty("uses_scsi_reservation") public String usesScsiReservation; - public Ref disk; + public Disk disk; public Ref vm; public String href; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ip.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ip.java new file mode 100644 index 000000000000..7afbc0710ffb --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ip.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Ip { + + private String address; + private String gateway; + private String netmask; + private String version; + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getGateway() { + return gateway; + } + + public void setGateway(String gateway) { + this.gateway = gateway; + } + + public String getNetmask() { + return netmask; + } + + public void setNetmask(String netmask) { + this.netmask = netmask; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java new file mode 100644 index 000000000000..11d94cc41791 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Ips { + + @JacksonXmlElementWrapper(useWrapping = false) + private List ip; + + public Ips(final List ip) { + this.ip = ip; + } + + public List getIp() { + return ip; + } + + public void setIp(List ip) { + this.ip = ip; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java new file mode 100644 index 000000000000..042d45c133d2 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Job { + private String autoCleared; + private String external; + private Long lastUpdated; + private Long startTime; + private Long endTime; + private String status; + private Ref owner; + private Actions actions; + private String description; + private List link; + private String href; + private String id; + + // getters and setters + public String getAutoCleared() { return autoCleared; } + public void setAutoCleared(String autoCleared) { this.autoCleared = autoCleared; } + + public String getExternal() { return external; } + public void setExternal(String external) { this.external = external; } + + public Long getLastUpdated() { return lastUpdated; } + public void setLastUpdated(Long lastUpdated) { this.lastUpdated = lastUpdated; } + + public Long getStartTime() { return startTime; } + public void setStartTime(Long startTime) { this.startTime = startTime; } + + public Long getEndTime() { return endTime; } + public void setEndTime(Long endTime) { this.endTime = endTime; } + + public String getStatus() { return status; } + public void setStatus(String status) { this.status = status; } + + public Ref getOwner() { return owner; } + public void setOwner(Ref owner) { this.owner = owner; } + + public Actions getActions() { return actions; } + public void setActions(Actions actions) { this.actions = actions; } + + public String getDescription() { return description; } + public void setDescription(String description) { this.description = description; } + + public List getLink() { return link; } + public void setLink(List link) { this.link = link; } + + public String getHref() { return href; } + public void setHref(String href) { this.href = href; } + + public String getId() { return id; } + public void setId(String id) { this.id = id; } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java new file mode 100644 index 000000000000..904950ae0a7a --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownershjob. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Jobs { + + @JacksonXmlElementWrapper(useWrapping = false) + private List job; + + public Jobs(final List job) { + this.job = job; + } + + public List getJob() { + return job; + } + + public void setJob(List job) { + this.job = job; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Mac.java similarity index 71% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Mac.java index 92547b337d5d..02d908054608 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmEntityResponse.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Mac.java @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.veeam.api.response; +package org.apache.cloudstack.veeam.api.dto; -import org.apache.cloudstack.veeam.api.dto.Vm; +import com.fasterxml.jackson.annotation.JsonInclude; -/** - * Required entity response: - * { "vm": { .. } } - */ -public final class VmEntityResponse { - public Vm vm; +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Mac { - public VmEntityResponse() {} + private String address; - public VmEntityResponse(final Vm vm) { - this.vm = vm; + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java new file mode 100644 index 000000000000..7eca9aff4f78 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Nic { + + private String href; + private String id; + private String name; + private String description; + @JacksonXmlProperty(localName = "interface") + @JsonProperty("interface") + private String interfaceType; + private String linked; + private Mac mac; + private String plugged; + private Ref vnicProfile; + private Ref vm; + private ReportedDevices reportedDevices; + + public Nic() { + } + + public String getHref() { + return href; + } + + public void setHref(final String href) { + this.href = href; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getInterfaceType() { + return interfaceType; + } + + public void setInterfaceType(String interfaceType) { + this.interfaceType = interfaceType; + } + + public boolean isLinked() { + return Boolean.parseBoolean(linked); + } + + public void setLinked(boolean linked) { + this.linked = Boolean.toString(linked); + } + + public Mac getMac() { + return mac; + } + + public void setMac(Mac mac) { + this.mac = mac; + } + + public boolean isPlugged() { + return Boolean.parseBoolean(plugged); + } + + public void setPlugged(boolean plugged) { + this.plugged = Boolean.toString(plugged); + } + + public Ref getVnicProfile() { + return vnicProfile; + } + + public void setVnicProfile(Ref vnicProfile) { + this.vnicProfile = vnicProfile; + } + + public Ref getVm() { + return vm; + } + + public void setVm(Ref vm) { + this.vm = vm; + } + + public ReportedDevices getReportedDevices() { + return reportedDevices; + } + + public void setReportedDevices(ReportedDevices reportedDevices) { + this.reportedDevices = reportedDevices; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java new file mode 100644 index 000000000000..37c0259fa53c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "nics") +public final class Nics { + + @JsonProperty("nic") + @JacksonXmlElementWrapper(useWrapping = false) + public List nic; + + public Nics() {} + + public Nics(final List nic) { + this.nic = nic; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java new file mode 100644 index 000000000000..7c36f2d02f5a --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +public class ReportedDevice { + private String comment; + private String description; + private Ips ips; + private String id; + private Mac Mac; + private String name; + private String type; + private Ref vm; + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Ips getIps() { + return ips; + } + + public void setIps(Ips ips) { + this.ips = ips; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Mac getMac() { + return Mac; + } + + public void setMac(Mac mac) { + Mac = mac; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Ref getVm() { + return vm; + } + + public void setVm(Ref vm) { + this.vm = vm; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java new file mode 100644 index 000000000000..7348b0ca6fa9 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ReportedDevices { + + @JacksonXmlElementWrapper(useWrapping = false) + private List reportedDevice; + + public ReportedDevices(final List reportedDevice) { + this.reportedDevice = reportedDevice; + } + + public List getReportedDevice() { + return reportedDevice; + } + + public void setReportedDevice(List reportedDevice) { + this.reportedDevice = reportedDevice; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 5a21f84c4ae5..c83a7536e6a2 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -46,6 +46,7 @@ public final class Vm { @JsonProperty("stop_time") @JacksonXmlProperty(localName = "stop_time") public Long stopTime; // epoch millis + private Long startTime; // epoch millis public Ref template; @@ -68,6 +69,43 @@ public final class Vm { public Actions actions; // actions.link[] @JacksonXmlElementWrapper(useWrapping = false) public List link; // related resources + public EmptyElement tags; // empty + private DiskAttachments diskAttachments; + private Nics nics; + + private VmInitialization initialization; public Vm() {} + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public DiskAttachments getDiskAttachments() { + return diskAttachments; + } + + public void setDiskAttachments(DiskAttachments diskAttachments) { + this.diskAttachments = diskAttachments; + } + + public Nics getNics() { + return nics; + } + + public void setNics(Nics nics) { + this.nics = nics; + } + + public VmInitialization getInitialization() { + return initialization; + } + + public void setInitialization(VmInitialization initialization) { + this.initialization = initialization; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java new file mode 100644 index 000000000000..9be7ab6891e5 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VmAction { + private Ref job; + private Vm vm; + private String status; + + public Ref getJob() { + return job; + } + + public void setJob(Ref job) { + this.job = job; + } + + public Vm getVm() { + return vm; + } + + public void setVm(Vm vm) { + this.vm = vm; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java new file mode 100644 index 000000000000..61982872afca --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VmInitialization { + + private String contentData; + + public String getContentData() { + return contentData; + } + + public void setContentData(String contentData) { + this.contentData = contentData; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java similarity index 86% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java index fc858f51ca05..df981129f1ce 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/VmCollectionResponse.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.veeam.api.response; +package org.apache.cloudstack.veeam.api.dto; import java.util.List; -import org.apache.cloudstack.veeam.api.dto.Vm; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -34,14 +32,14 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ "vm" }) @JacksonXmlRootElement(localName = "vms") -public final class VmCollectionResponse { +public final class Vms { @JsonProperty("vm") @JacksonXmlElementWrapper(useWrapping = false) public List vm; - public VmCollectionResponse() {} + public Vms() {} - public VmCollectionResponse(final List vm) { + public Vms(final List vm) { this.vm = vm; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java index 0d6af22599e4..933e57b202a3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.dataformat.xml.XmlMapper; @@ -38,6 +39,7 @@ public Mapper() { private static void configure(final ObjectMapper mapper) { mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); mapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); // If you ever add enums etc: // mapper.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); // mapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index b247550cf140..f56a19d84715 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -42,11 +42,12 @@ + - + diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index a72498c13718..fed8de36c3de 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -204,5 +204,5 @@ static Set getStrictHostTags() { */ boolean isVMPartOfAnyCKSCluster(VMInstanceVO vm); - boolean isDummyTemplate(HypervisorType hypervisorType, Long templateId); + boolean isBlankInstanceTemplate(VirtualMachineTemplate template); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 02a16e7a9e45..3fa6cd105c9b 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -423,7 +423,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private static final long GiB_TO_BYTES = 1024 * 1024 * 1024; - public static final String KVM_VM_DUMMY_TEMPLATE_NAME = "kvm-vm-dummy-template"; + private static final String KVM_VM_DUMMY_TEMPLATE_NAME = "kvm-vm-dummy-template"; @Inject @@ -658,7 +658,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private boolean _instanceNameFlag; private int _scaleRetry; private Map vmIdCountMap = new ConcurrentHashMap<>(); - private static VMTemplateVO KVM_VM_DUMMY_TEMPLATE; protected static long ROOT_DEVICE_ID = 0; @@ -2505,17 +2504,6 @@ public boolean configure(String name, Map params) throws Configu _scaleRetry = NumbersUtil.parseInt(configs.get(Config.ScaleRetry.key()), 2); _vmIpFetchThreadExecutor = Executors.newFixedThreadPool(VmIpFetchThreadPoolMax.value(), new NamedThreadFactory("vmIpFetchThread")); - - KVM_VM_DUMMY_TEMPLATE = _templateDao.findByAccountAndName(Account.ACCOUNT_ID_SYSTEM, KVM_VM_DUMMY_TEMPLATE_NAME); - if (KVM_VM_DUMMY_TEMPLATE == null) { - KVM_VM_DUMMY_TEMPLATE = VMTemplateVO.createSystemIso(_templateDao.getNextInSequence(Long.class, "id"), KVM_VM_DUMMY_TEMPLATE_NAME, KVM_VM_DUMMY_TEMPLATE_NAME, true, - "", true, 64, Account.ACCOUNT_ID_SYSTEM, "", - "Dummy Template for KVM VM", false, 1); - KVM_VM_DUMMY_TEMPLATE.setState(VirtualMachineTemplate.State.Active); - KVM_VM_DUMMY_TEMPLATE.setFormat(ImageFormat.QCOW2); - KVM_VM_DUMMY_TEMPLATE = _templateDao.persist(KVM_VM_DUMMY_TEMPLATE); - } - logger.info("User VM Manager is configured."); return true; @@ -3945,7 +3933,7 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service _accountMgr.checkAccess(owner, _diskOfferingDao.findById(diskOfferingId), zone); // If no network is specified, find system security group enabled network - if (isDummyTemplate(hypervisor, template.getId())) { + if (isBlankInstanceTemplate(template)) { logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced security group enabled zone", hypervisor); } else if (networkIdList == null || networkIdList.isEmpty()) { Network networkWithSecurityGroup = _networkModel.getNetworkWithSGWithFreeIPs(owner, zone.getId()); @@ -4060,7 +4048,7 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv _accountMgr.checkAccess(owner, diskOffering, zone); List vpcSupportedHTypes = _vpcMgr.getSupportedVpcHypervisors(); - if (isDummyTemplate(hypervisor, template.getId())) { + if (isBlankInstanceTemplate(template)) { logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced zone", hypervisor); } else if (networkIdList == null || networkIdList.isEmpty()) { NetworkVO defaultNetwork = getDefaultNetwork(zone, owner, false); @@ -4497,7 +4485,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri } } - if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType) && !isDummyTemplate(hypervisorType, template.getId())) { + if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType) && !isBlankInstanceTemplate(template)) { throw new InvalidParameterValueException(String.format("Unable to use system template %s to deploy a user vm", template)); } @@ -4510,7 +4498,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri if (CollectionUtils.isEmpty(snapshotsOnZone)) { throw new InvalidParameterValueException("The snapshot does not exist on zone " + zone.getId()); } - } else if (!isDummyTemplate(hypervisorType, template.getId())) { + } else if (!isBlankInstanceTemplate(template)) { List listZoneTemplate = _templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { throw new InvalidParameterValueException("The template " + template.getId() + " is not available for use"); @@ -4625,7 +4613,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri // by Agent Manager in order to configure default // gateway for the vm if (defaultNetworkNumber == 0) { - if (isDummyTemplate(hypervisorType, template.getId())) { + if (isBlankInstanceTemplate(template)) { logger.debug("Template is a dummy template for hypervisor {}, vm can be created without a default network", hypervisorType); } else { throw new InvalidParameterValueException("At least 1 default network has to be specified for the vm"); @@ -4791,7 +4779,7 @@ protected long configureCustomRootDiskSize(Map customParameters, return rootDiskSize; } else { // For baremetal, size can be 0 (zero) - Long templateSize = _templateDao.findById(template.getId()).getSize(); + Long templateSize = template.getSize(); if (templateSize != null) { return templateSize; } @@ -5347,7 +5335,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "deploying Vm", async = true) public UserVm startVirtualMachine(DeployVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { long vmId = cmd.getEntityId(); - if (!cmd.getStartVm() || cmd.getDummy()) { + if (!cmd.getStartVm() || cmd.isBlankInstance()) { return getUserVm(vmId); } Long podId = null; @@ -6495,10 +6483,10 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE (!(HypervisorType.KVM.equals(template.getHypervisorType()) || HypervisorType.KVM.equals(cmd.getHypervisor())))) { throw new InvalidParameterValueException("Deploying a virtual machine with existing volume/snapshot is supported only from KVM hypervisors"); } - if (template == null && HypervisorType.KVM.equals(cmd.getHypervisor()) && cmd.getDummy()) { - template = KVM_VM_DUMMY_TEMPLATE; + if (template == null && HypervisorType.KVM.equals(cmd.getHypervisor()) && cmd.isBlankInstance()) { + template = getBlankInstanceTemplate(); logger.info("Creating launch permission for Dummy template"); - LaunchPermissionVO launchPermission = new LaunchPermissionVO(KVM_VM_DUMMY_TEMPLATE.getId(), owner.getId()); + LaunchPermissionVO launchPermission = new LaunchPermissionVO(template.getId(), owner.getId()); launchPermissionDao.persist(launchPermission); } // Make sure a valid template ID was specified @@ -6660,9 +6648,9 @@ private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Accoun applyLeaseOnCreateInstance(vm, cmd.getLeaseDuration(), cmd.getLeaseExpiryAction(), svcOffering); } - if (KVM_VM_DUMMY_TEMPLATE != null && template.getId() == KVM_VM_DUMMY_TEMPLATE.getId() && cmd instanceof DeployVMCmd && ((DeployVMCmd) cmd).getDummy()) { + if (isBlankInstanceTemplate(template) && cmd instanceof DeployVMCmd && ((DeployVMCmd) cmd).isBlankInstance()) { logger.info("Revoking launch permission for Dummy template"); - launchPermissionDao.removePermissions(KVM_VM_DUMMY_TEMPLATE.getId(), Collections.singletonList(owner.getId())); + launchPermissionDao.removePermissions(template.getId(), Collections.singletonList(owner.getId())); } return vm; @@ -10101,10 +10089,23 @@ private void setVncPasswordForKvmIfAvailable(Map customParameter } @Override - public boolean isDummyTemplate(HypervisorType hypervisorType, Long templateId) { - if (HypervisorType.KVM.equals(hypervisorType) && KVM_VM_DUMMY_TEMPLATE != null && KVM_VM_DUMMY_TEMPLATE.getId() == templateId) { - return true; - } - return false; + public boolean isBlankInstanceTemplate(VirtualMachineTemplate template) { + return KVM_VM_DUMMY_TEMPLATE_NAME.equals(template.getUniqueName()); + } + + VMTemplateVO getBlankInstanceTemplate() { + VMTemplateVO template = _templateDao.findByName(KVM_VM_DUMMY_TEMPLATE_NAME); + if (template != null) { + return template; + } + template = VMTemplateVO.createSystemIso(_templateDao.getNextInSequence(Long.class, "id"), + KVM_VM_DUMMY_TEMPLATE_NAME, KVM_VM_DUMMY_TEMPLATE_NAME, true, + "", true, 64, Account.ACCOUNT_ID_SYSTEM, "", + "Dummy Template for KVM VM", false, 1); + template.setState(VirtualMachineTemplate.State.Active); + template.setFormat(ImageFormat.QCOW2); + template = _templateDao.persist(template); +// _templateDao.remove(template.getId()); + return template; } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index e5894430fbfe..ca0de8227694 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -478,6 +478,16 @@ public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTran return imageTransferDao.findById(imageTransfer.getId()); } + @Override + public boolean cancelImageTransfer(long imageTransferId) { + ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); + if (imageTransfer == null) { + throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); + } + // ToDo: Implement cancel logic + return true; + } + private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); @@ -552,8 +562,11 @@ private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { @Override public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { - Long imageTransferId = cmd.getImageTransferId(); + return finalizeImageTransfer(cmd.getImageTransferId()); + } + @Override + public boolean finalizeImageTransfer(final long imageTransferId) { ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); if (imageTransfer == null) { throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); @@ -566,6 +579,8 @@ public boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd) { } imageTransfer.setPhase(ImageTransferVO.Phase.finished); imageTransferDao.update(imageTransfer.getId(), imageTransfer); +// ToDo: check this +// imageTransferDao.remove(imageTransfer.getId()); return true; } @@ -656,7 +671,8 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans response.setBackupId(backup.getUuid()); } Long volumeId = imageTransferVO.getDiskId(); - Volume volume = volumeDao.findById(volumeId); + // ToDo: fix volume deletion leaving orphan image transfer record + Volume volume = volumeDao.findByIdIncludingRemoved(volumeId); response.setDiskId(volume.getUuid()); response.setTransferUrl(imageTransferVO.getTransferUrl()); response.setPhase(imageTransferVO.getPhase().toString()); From 586134d392081502ed24605b2e630910110c7909 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:01:29 +0530 Subject: [PATCH 026/129] Support file backend for cow format: api and server --- .../admin/backup/CreateImageTransferCmd.java | 13 +- .../cloudstack/backup/ImageTransfer.java | 12 ++ .../backup/IncrementalBackupService.java | 2 +- .../backup/CreateImageTransferCommand.java | 28 ++++- .../cloudstack/backup/ImageTransferVO.java | 29 ++++- .../backup/dao/ImageTransferDao.java | 1 + .../backup/dao/ImageTransferDaoImpl.java | 14 +++ .../META-INF/db/schema-42100to42200.sql | 5 +- .../META-INF/db/schema-42210to42300.sql | 2 + .../veeam/adapter/ServerAdapter.java | 8 +- .../backup/IncrementalBackupServiceImpl.java | 115 ++++++++++++------ 11 files changed, 179 insertions(+), 50 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index b67128e47dce..c50a914cd13e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -32,6 +32,8 @@ import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.context.CallContext; +import com.cloud.utils.EnumUtils; + @APICommand(name = "createImageTransfer", description = "Create image transfer for a disk in backup", responseObject = ImageTransferResponse.class, @@ -61,6 +63,11 @@ public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { description = "Direction of the transfer: upload, download") private String direction; + @Parameter(name = ApiConstants.FORMAT, + type = CommandType.STRING, + description = "Format of the image: cow/raw. Currently only raw is supported for download. Defaults to raw if not provided") + private String format; + public Long getBackupId() { return backupId; } @@ -73,7 +80,11 @@ public ImageTransfer.Direction getDirection() { return ImageTransfer.Direction.valueOf(direction); } - @Override + public ImageTransfer.Format getFormat() { + return EnumUtils.fromString(ImageTransfer.Format.class, format); + } + + @Override public void execute() { ImageTransferResponse response = incrementalBackupService.createImageTransfer(this); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java index ca6b546e04fc..cf09749bcfc6 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -27,6 +27,16 @@ public enum Direction { upload, download } + public enum Format { + raw, + cow + } + + public enum Backend { + nbd, + file + } + public enum Phase { initializing, transferring, finished, failed } @@ -47,5 +57,7 @@ public enum Phase { Direction getDirection(); + Backend getBackend(); + String getSignedTicketId(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index c37aa5b89eec..67ef7175c416 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -62,7 +62,7 @@ public interface IncrementalBackupService extends Configurable, PluggableService */ ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); - ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction); + ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format); boolean cancelImageTransfer(long imageTransferId); diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 43bde925f755..4fb8743b6252 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -26,19 +26,35 @@ public class CreateImageTransferCommand extends Command { private int nbdPort; private String direction; private String checkpointId; + private String file; + private ImageTransfer.Backend backend; public CreateImageTransferCommand() { } - public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, int nbdPort, String direction, String checkpointId) { + private CreateImageTransferCommand(String transferId, String hostIpAddress, String direction) { this.transferId = transferId; this.hostIpAddress = hostIpAddress; + this.direction = direction; + } + + public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String exportName, int nbdPort, String checkpointId) { + this(transferId, hostIpAddress, direction); + this.backend = ImageTransfer.Backend.nbd; this.exportName = exportName; this.nbdPort = nbdPort; - this.direction = direction; this.checkpointId = checkpointId; } + public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String file) { + this(transferId, hostIpAddress, direction); + if (direction == ImageTransfer.Direction.download.toString()) { + throw new IllegalArgumentException("File backend is only supported for upload"); + } + this.backend = ImageTransfer.Backend.file; + this.file = file; + } + public String getExportName() { return exportName; } @@ -47,6 +63,14 @@ public int getNbdPort() { return nbdPort; } + public String getFile() { + return file; + } + + public ImageTransfer.Backend getBackend() { + return backend; + } + public String getHostIpAddress() { return hostIpAddress; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index a6c5bce07d76..6562ba74a777 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -54,6 +54,9 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "nbd_port") private int nbdPort; + @Column(name = "file") + private String file; + @Column(name = "transfer_url") private String transferUrl; @@ -65,6 +68,10 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "direction") private Direction direction; + @Enumerated(value = EnumType.STRING) + @Column(name = "backend") + private Backend backend; + @Column(name = "signed_ticket_id") private String signedTicketId; @@ -95,12 +102,10 @@ public class ImageTransferVO implements ImageTransfer { public ImageTransferVO() { } - public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + private ImageTransferVO(String uuid, long diskId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { this.uuid = uuid; - this.backupId = backupId; this.diskId = diskId; this.hostId = hostId; - this.nbdPort = nbdPort; this.phase = phase; this.direction = direction; this.accountId = accountId; @@ -109,6 +114,19 @@ public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int this.created = new Date(); } + public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.backupId = backupId; + this.nbdPort = nbdPort; + this.backend = Backend.nbd; + } + + public ImageTransferVO(String uuid, long diskId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.file = file; + this.backend = Backend.file; + } + @Override public long getId() { return id; @@ -183,6 +201,11 @@ public void setDirection(Direction direction) { this.direction = direction; } + @Override + public Backend getBackend() { + return backend; + } + @Override public String getSignedTicketId() { return signedTicketId; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index 035e22958e5e..e8c30d27ee79 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -29,5 +29,6 @@ public interface ImageTransferDao extends GenericDao { ImageTransferVO findByUuid(String uuid); ImageTransferVO findByNbdPort(int port); ImageTransferVO findByVolume(Long volumeId); + ImageTransferVO findUnfinishedByVolume(Long volumeId); List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index e7d87446326c..7e311d2a00fe 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -36,6 +36,7 @@ public class ImageTransferDaoImpl extends GenericDaoBase private SearchBuilder uuidSearch; private SearchBuilder nbdPortSearch; private SearchBuilder volumeSearch; + private SearchBuilder volumeUnfinishedSearch; private SearchBuilder phaseDirectionSearch; public ImageTransferDaoImpl() { @@ -59,6 +60,11 @@ protected void init() { volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ); volumeSearch.done(); + volumeUnfinishedSearch = createSearchBuilder(); + volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getDiskId(), SearchCriteria.Op.EQ); + volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ); + volumeUnfinishedSearch.done(); + phaseDirectionSearch = createSearchBuilder(); phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ); phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ); @@ -93,6 +99,14 @@ public ImageTransferVO findByVolume(Long volumeId) { return findOneBy(sc); } + @Override + public ImageTransferVO findUnfinishedByVolume(Long volumeId) { + SearchCriteria sc = volumeUnfinishedSearch.create(); + sc.setParameters("volumeId", volumeId); + sc.setParameters("phase", ImageTransferVO.Phase.finished.toString()); + return findOneBy(sc); + } + @Override public List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) { SearchCriteria sc = phaseDirectionSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index d9f2ccd70cea..1e2654213870 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -18,7 +18,7 @@ --; -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; - +not supported for download -- health check status as enum CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); @@ -93,3 +93,6 @@ UPDATE `cloud`.`configuration` SET `scope` = 2 WHERE `name` = 'use.https.to.uplo -- Delete the configuration for 'use.https.to.upload' from StoragePool DELETE FROM `cloud`.`storage_pool_details` WHERE `name` = 'use.https.to.upload'; +<<<<<<< HEAD +======= +>>>>>>> 1ec4e52fa6 (Support file backend for cow format: api and server) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 3a2bbf0bd5b1..f81e2904841f 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -141,8 +141,10 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', `nbd_port` int NOT NULL COMMENT 'NBD port', `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', + `file` varchar(255) COMMENT 'File for the file backend', `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file', `progress` int COMMENT 'Transfer progress percentage (0-100)', `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', `created` datetime NOT NULL COMMENT 'date created', diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 0cb2b56d0718..f4fff169c487 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -53,6 +53,7 @@ import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.backup.ImageTransfer.Direction; +import org.apache.cloudstack.backup.ImageTransfer.Format; import org.apache.cloudstack.backup.ImageTransferVO; import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.backup.dao.ImageTransferDao; @@ -753,7 +754,8 @@ public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { if (direction == null) { throw new InvalidParameterValueException("Invalid or missing direction"); } - return createImageTransfer(null, volumeVO.getId(), direction); + Format format = EnumUtils.fromString(Format.class, request.getFormat()); + return createImageTransfer(null, volumeVO.getId(), direction, format); } public boolean handleCancelImageTransfer(String uuid) { @@ -772,12 +774,12 @@ public boolean handleFinalizeImageTransfer(String uuid) { return incrementalBackupService.finalizeImageTransfer(vo.getId()); } - private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction) { + private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { Account serviceAccount = createServiceAccountIfNeeded(); CallContext.register(serviceAccount.getId(), serviceAccount.getId()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = - incrementalBackupService.createImageTransfer(volumeId, null, direction); + incrementalBackupService.createImageTransfer(volumeId, null, direction, format); ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); } finally { diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index ca0de8227694..b2e906aed4fc 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -50,7 +50,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; @@ -251,8 +250,11 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); List transfers = imageTransferDao.listByBackupId(backupId); - if (CollectionUtils.isNotEmpty(transfers)) { - throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId); + for (ImageTransferVO transfer : transfers) { + if (transfer.getPhase() != ImageTransferVO.Phase.finished) { + throw new CloudRuntimeException(String.format("Image transfer %s not finalized for backup: %s", transfer.getUuid(), backup.getUuid())); + } + imageTransferDao.remove(transfer.getId()); } if (vm.getState() == State.Running) { @@ -294,13 +296,16 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } - private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume) { + private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume, ImageTransfer.Backend backend) { final String direction = ImageTransfer.Direction.download.toString(); BackupVO backup = backupDao.findById(backupId); if (backup == null) { throw new CloudRuntimeException("Backup not found: " + backupId); } boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); + if (ImageTransfer.Backend.file.equals(backend)) { + throw new CloudRuntimeException("File backend is not supported for download"); + } String transferId = UUID.randomUUID().toString(); Host host = hostDao.findById(backup.getHostId()); @@ -314,11 +319,10 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( transferId, host.getPrivateIpAddress(), + direction, volume.getUuid(), backup.getNbdPort(), - direction, - backup.getFromCheckpointId() - ); + backup.getFromCheckpointId()); try { CreateImageTransferAnswer answer; @@ -396,50 +400,70 @@ private String getVolumePathForFileBasedBackend(Volume volume) { return volumePath; } - private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { + private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer.Backend backend) { final String direction = ImageTransfer.Direction.upload.toString(); String transferId = UUID.randomUUID().toString(); - int nbdPort = allocateNbdPort(); Long poolId = volume.getPoolId(); StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); Host host = getFirstHostFromStoragePool(storagePoolVO); String volumePath = getVolumePathForFileBasedBackend(volume); - startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort); + ImageTransferVO imageTransfer; + CreateImageTransferCommand transferCmd; + if (backend.equals(ImageTransfer.Backend.file)) { + imageTransfer = new ImageTransferVO( + transferId, + volume.getId(), + host.getId(), + volumePath, + ImageTransferVO.Phase.transferring, + ImageTransfer.Direction.upload, + volume.getAccountId(), + volume.getDomainId(), + volume.getDataCenterId()); - ImageTransferVO imageTransfer = new ImageTransferVO( - transferId, - null, - volume.getId(), - host.getId(), - nbdPort, - ImageTransferVO.Phase.transferring, - ImageTransfer.Direction.upload, - volume.getAccountId(), - volume.getDomainId(), - volume.getDataCenterId() - ); + transferCmd = new CreateImageTransferCommand( + transferId, + host.getPrivateIpAddress(), + direction, + volumePath); + + } else { + int nbdPort = allocateNbdPort(); + startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort); + imageTransfer = new ImageTransferVO( + transferId, + null, + volume.getId(), + host.getId(), + nbdPort, + ImageTransferVO.Phase.transferring, + ImageTransfer.Direction.upload, + volume.getAccountId(), + volume.getDomainId(), + volume.getDataCenterId()); + + transferCmd = new CreateImageTransferCommand( + transferId, + host.getPrivateIpAddress(), + direction, + volume.getUuid(), + nbdPort, + null); + } - CreateImageTransferAnswer transferAnswer; - CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( - transferId, - host.getPrivateIpAddress(), - volume.getUuid(), - nbdPort, - direction, - null - ); EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId()); - transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); + CreateImageTransferAnswer transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); if (!transferAnswer.getResult()) { - stopNbdServer(imageTransfer); + if (!backend.equals(ImageTransfer.Backend.file)) { + stopNbdServer(imageTransfer); + } throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); } - imageTransfer.setTransferUrl(transferAnswer.getTransferUrl()); imageTransfer.setSignedTicketId(transferAnswer.getImageTransferId()); imageTransfer = imageTransferDao.persist(imageTransfer); @@ -447,9 +471,21 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { } + private ImageTransfer.Backend getImageTransferBackend(ImageTransfer.Format format, ImageTransfer.Direction direction) { + if (ImageTransfer.Format.cow.equals(format)) { + if (ImageTransfer.Direction.download.equals(direction)) { + logger.debug("Using NBD backend for download"); + return ImageTransfer.Backend.nbd; + } + return ImageTransfer.Backend.file; + } else { + return ImageTransfer.Backend.nbd; + } + } + @Override public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { - ImageTransfer imageTransfer = createImageTransfer(cmd.getVolumeId(), cmd.getBackupId(), cmd.getDirection()); + ImageTransfer imageTransfer = createImageTransfer(cmd.getVolumeId(), cmd.getBackupId(), cmd.getDirection(), cmd.getFormat()); if (imageTransfer instanceof ImageTransferVO) { ImageTransferVO imageTransferVO = (ImageTransferVO) imageTransfer; return toImageTransferResponse(imageTransferVO); @@ -458,19 +494,20 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { } @Override - public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction) { + public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format) { ImageTransfer imageTransfer; VolumeVO volume = volumeDao.findById(volumeId); - ImageTransferVO existingTransfer = imageTransferDao.findByVolume(volume.getId()); + ImageTransferVO existingTransfer = imageTransferDao.findUnfinishedByVolume(volume.getId()); if (existingTransfer != null) { throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid()); } + ImageTransfer.Backend backend = getImageTransferBackend(format, direction); if (ImageTransfer.Direction.upload.equals(direction)) { - imageTransfer = createUploadImageTransfer(volume); + imageTransfer = createUploadImageTransfer(volume, backend); } else if (ImageTransfer.Direction.download.equals(direction)) { - imageTransfer = createDownloadImageTransfer(backupId, volume); + imageTransfer = createDownloadImageTransfer(backupId, volume, backend); } else { throw new CloudRuntimeException("Invalid direction: " + direction); } From 6ca1c9b31fe9f8f844c850e309ba57cf76e9dde8 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 8 Feb 2026 14:41:48 +0530 Subject: [PATCH 027/129] Image server support for file backend (qcow2 upload) --- .../resource/NfsSecondaryStorageResource.java | 35 ++- systemvm/debian/opt/cloud/bin/image_server.py | 239 ++++++++++++++---- 2 files changed, 207 insertions(+), 67 deletions(-) diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 458eb32ca890..2358bdcc8324 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -57,6 +57,7 @@ import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.FinalizeImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.framework.security.keystore.KeystoreManager; import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -3839,10 +3840,8 @@ protected Answer execute(CreateImageTransferCommand cmd) { } final String transferId = cmd.getTransferId(); - final String hostIp = cmd.getHostIpAddress(); - final String exportName = cmd.getExportName(); - final int nbdPort = cmd.getNbdPort(); + final ImageTransfer.Backend backend = cmd.getBackend(); if (StringUtils.isBlank(transferId)) { return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); @@ -3850,18 +3849,25 @@ protected Answer execute(CreateImageTransferCommand cmd) { if (StringUtils.isBlank(hostIp)) { return new CreateImageTransferAnswer(cmd, false, "hostIpAddress is empty."); } - if (StringUtils.isBlank(exportName)) { - return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); - } - if (nbdPort <= 0) { - return new CreateImageTransferAnswer(cmd, false, "Invalid nbdPort: " + nbdPort); - } - final int imageServerPort = 54323; + final Map payload = new HashMap<>(); + payload.put("backend", backend.toString()); - try { - // 1) Write /tmp/ with NBD endpoint details. - final Map payload = new HashMap<>(); + if (backend == ImageTransfer.Backend.file) { + final String filePath = cmd.getFile(); + if (StringUtils.isBlank(filePath)) { + return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); + } + payload.put("file", filePath); + } else { + final String exportName = cmd.getExportName(); + final int nbdPort = cmd.getNbdPort(); + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); + } + if (nbdPort <= 0) { + return new CreateImageTransferAnswer(cmd, false, "Invalid nbdPort: " + nbdPort); + } payload.put("host", hostIp); payload.put("port", nbdPort); payload.put("export", exportName); @@ -3869,7 +3875,9 @@ protected Answer execute(CreateImageTransferCommand cmd) { if (checkpointId != null) { payload.put("export_bitmap", exportName + "-" + checkpointId.substring(0, 4)); } + } + try { final String json = new GsonBuilder().create().toJson(payload); File dir = new File("/tmp/imagetransfer"); if (!dir.exists()) { @@ -3883,6 +3891,7 @@ protected Answer execute(CreateImageTransferCommand cmd) { return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on SSVM: " + e.getMessage()); } + final int imageServerPort = 54323; startImageServerIfNotRunning(imageServerPort); final String transferUrl = String.format("http://%s:%d/images/%s", _publicIp, imageServerPort, transferId); diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 848eb41983c0..a176513698c5 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -17,7 +17,11 @@ # under the License. """ -POC "imageio-like" HTTP server backed by NBD over TCP. +POC "imageio-like" HTTP server backed by NBD over TCP or a local file. + +Supports two backends (see config payload): +- nbd: proxy to an NBD server (port, export, export_bitmap); supports range reads/writes, extents, zero, flush. +- file: read/write a local qcow2 (or raw) file path; full PUT only (no range writes), GET with optional ranges, flush. How to run ---------- @@ -116,9 +120,10 @@ _IMAGE_LOCKS_GUARD = threading.Lock() -# Dynamic image_id(transferId) -> NBD export mapping: +# Dynamic image_id(transferId) -> backend mapping: # CloudStack writes a JSON file at /tmp/imagetransfer/ with: -# {"host": "...", "port": 10809, "export": "vda", "export_bitmap":"bitmap1"} +# - NBD backend: {"backend": "nbd", "host": "...", "port": 10809, "export": "vda", "export_bitmap": "..."} +# - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} # # This server reads that file on-demand. _CFG_DIR = "/tmp/imagetransfer" @@ -249,26 +254,49 @@ def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) return None - host = obj.get("host") - port = obj.get("port") - export = obj.get("export") - export_bitmap = obj.get("export_bitmap") - if not isinstance(host, str) or not host: - logging.error("cfg missing/invalid host image_id=%s", image_id) - return None - try: - port_i = int(port) - except Exception: - logging.error("cfg missing/invalid port image_id=%s", image_id) + backend = obj.get("backend") + if backend is None: + backend = "nbd" + if not isinstance(backend, str): + logging.error("cfg invalid backend type image_id=%s", image_id) return None - if port_i <= 0 or port_i > 65535: - logging.error("cfg out-of-range port image_id=%s port=%r", image_id, port) - return None - if export is not None and (not isinstance(export, str) or not export): - logging.error("cfg missing/invalid export image_id=%s", image_id) + backend = backend.lower() + if backend not in ("nbd", "file"): + logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) return None - cfg: Dict[str, Any] = {"host": host, "port": port_i, "export": export, "export_bitmap": export_bitmap} + if backend == "file": + file_path = obj.get("file") + if not isinstance(file_path, str) or not file_path.strip(): + logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) + return None + cfg = {"backend": "file", "file": file_path.strip()} + else: + host = obj.get("host") + port = obj.get("port") + export = obj.get("export") + export_bitmap = obj.get("export_bitmap") + if not isinstance(host, str) or not host: + logging.error("cfg missing/invalid host image_id=%s", image_id) + return None + try: + port_i = int(port) + except Exception: + logging.error("cfg missing/invalid port image_id=%s", image_id) + return None + if port_i <= 0 or port_i > 65535: + logging.error("cfg out-of-range port image_id=%s port=%r", image_id, port) + return None + if export is not None and (not isinstance(export, str) or not export): + logging.error("cfg missing/invalid export image_id=%s", image_id) + return None + cfg = { + "backend": "nbd", + "host": host, + "port": port_i, + "export": export, + "export_bitmap": export_bitmap, + } with _CFG_CACHE_GUARD: _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) @@ -813,6 +841,9 @@ def _parse_query(self) -> Dict[str, List[str]]: def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: return _load_image_cfg(image_id) + def _is_file_backend(self, cfg: Dict[str, Any]) -> bool: + return cfg.get("backend") == "file" + def do_OPTIONS(self) -> None: image_id, tail = self._parse_route() if image_id is None or tail is not None: @@ -822,6 +853,19 @@ def do_OPTIONS(self) -> None: if cfg is None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return + if self._is_file_backend(cfg): + # File backend: full PUT only, no range writes; GET with ranges allowed; flush supported. + allowed_methods = "GET, PUT, POST, OPTIONS" + features = ["flush"] + max_writers = MAX_PARALLEL_WRITES + response = { + "unix_socket": None, + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": max_writers, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + return # Query NBD backend for capabilities (like nbdinfo); fall back to config. read_only = True can_flush = False @@ -876,6 +920,11 @@ def do_GET(self) -> None: return if tail == "extents": + if self._is_file_backend(cfg): + self._send_error_json( + HTTPStatus.BAD_REQUEST, "extents not supported for file backend" + ) + return query = self._parse_query() context = (query.get("context") or [None])[0] self._handle_get_extents(image_id, cfg, context=context) @@ -945,6 +994,12 @@ def do_PATCH(self) -> None: if cfg is None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return + if self._is_file_backend(cfg): + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "range writes and PATCH not supported for file backend; use PUT for full upload", + ) + return content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() range_header = self.headers.get("Range") @@ -1057,9 +1112,14 @@ def _handle_get_image( bytes_sent = 0 try: logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - size = conn.size() - + if self._is_file_backend(cfg): + file_path = cfg["file"] + try: + size = os.path.getsize(file_path) + except OSError as e: + logging.error("GET file size error image_id=%s path=%s err=%r", image_id, file_path, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "failed to access file") + return start_off = 0 end_off_incl = size - 1 if size > 0 else -1 status = HTTPStatus.OK @@ -1089,18 +1149,65 @@ def _handle_get_image( offset = start_off end_excl = end_off_incl + 1 - while offset < end_excl: - to_read = min(CHUNK_SIZE, end_excl - offset) - data = conn.pread(to_read, offset) - if not data: - raise RuntimeError("backend returned empty read") - try: - self.wfile.write(data) - except BrokenPipeError: - logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) - break - offset += len(data) - bytes_sent += len(data) + with open(file_path, "rb") as f: + f.seek(offset) + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = f.read(to_read) + if not data: + break + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) + else: + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + size = conn.size() + + start_off = 0 + end_off_incl = size - 1 if size > 0 else -1 + status = HTTPStatus.OK + content_length = size + if range_header is not None: + try: + start_off, end_off_incl = self._parse_single_range(range_header, size) + except ValueError as e: + if str(e) == "unsatisfiable": + self._send_range_not_satisfiable(size) + return + if "unsatisfiable" in str(e): + self._send_range_not_satisfiable(size) + return + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") + return + status = HTTPStatus.PARTIAL_CONTENT + content_length = (end_off_incl - start_off) + 1 + + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(content_length)) + if status == HTTPStatus.PARTIAL_CONTENT: + self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") + self.end_headers() + + offset = start_off + end_excl = end_off_incl + 1 + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = conn.pread(to_read, offset) + if not data: + raise RuntimeError("backend returned empty read") + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) except Exception as e: # If headers already sent, we can't return JSON reliably; just log. logging.error("GET error image_id=%s err=%r", image_id, e) @@ -1132,24 +1239,41 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: bytes_written = 0 try: logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - offset = 0 + if self._is_file_backend(cfg): + file_path = cfg["file"] remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {offset} bytes", - ) - return - conn.pwrite(chunk, offset) - offset += len(chunk) - remaining -= len(chunk) - bytes_written += len(chunk) - - # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. + with open(file_path, "wb") as f: + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {bytes_written} bytes", + ) + return + f.write(chunk) + bytes_written += len(chunk) + remaining -= len(chunk) self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + else: + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + offset = 0 + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {offset} bytes", + ) + return + conn.pwrite(chunk, offset) + offset += len(chunk) + remaining -= len(chunk) + bytes_written += len(chunk) + + # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) except Exception as e: logging.error("PUT error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") @@ -1244,9 +1368,16 @@ def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: start = _now_s() try: logging.info("FLUSH start image_id=%s", image_id) - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - conn.flush() - self._send_json(HTTPStatus.OK, {"ok": True}) + if self._is_file_backend(cfg): + file_path = cfg["file"] + with open(file_path, "rb") as f: + f.flush() + os.fsync(f.fileno()) + self._send_json(HTTPStatus.OK, {"ok": True}) + else: + with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: + conn.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) except Exception as e: logging.error("FLUSH error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") From fba7c634cbc33818e287638b50f00db6f0249e97 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 8 Feb 2026 16:37:51 +0530 Subject: [PATCH 028/129] ut failure in UserVmManagerImplTest --- server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 1a38c1b0a06f..cd102a07ee5c 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -846,9 +846,7 @@ public void configureCustomRootDiskSizeTestEmptyParametersAndOfferingRootSize() private void prepareAndRunConfigureCustomRootDiskSizeTest(Map customParameters, long expectedRootDiskSize, int timesVerifyIfHypervisorSupports, Long offeringRootDiskSize) { VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - Mockito.when(template.getId()).thenReturn(1l); Mockito.when(template.getSize()).thenReturn(99L * GiB_TO_BYTES); - Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(template); DiskOfferingVO diskfferingVo = Mockito.mock(DiskOfferingVO.class); From a89f872b4f4871c4eab641469a3d5090d0dd94e0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 10 Feb 2026 09:38:32 +0530 Subject: [PATCH 029/129] wip Signed-off-by: Abhishek Kumar --- .../cloudstack/api/ApiServerService.java | 19 ++ .../veeam/adapter/ServerAdapter.java | 124 ++++++++---- .../veeam/api/JobsRouteHandler.java | 2 +- .../AsyncJobJoinVOToJobConverter.java | 41 ++++ .../main/java/com/cloud/api/ApiServer.java | 188 +++++++++--------- 5 files changed, 245 insertions(+), 129 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java index 18c96c371591..1ee41ac86c22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java @@ -21,8 +21,11 @@ import javax.servlet.http.HttpSession; +import org.apache.cloudstack.context.CallContext; + import com.cloud.domain.Domain; import com.cloud.exception.CloudAuthenticationException; +import com.cloud.user.Account; import com.cloud.user.UserAccount; public interface ApiServerService { @@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas String getDomainId(Map params); boolean isPostRequestsAndTimestampsEnforced(); + + AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map params, CallContext ctx, Long callerUserId, Account caller) throws Exception; + + class AsyncCmdResult { + public final Long objectId; + public final String objectUuid; + public final BaseAsyncCmd asyncCmd; + public final long jobId; + + public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) { + this.objectId = objectId; + this.objectUuid = objectUuid; + this.asyncCmd = asyncCmd; + this.jobId = jobId; + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index f4fff169c487..b7d8e2699766 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -33,6 +34,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.Rule; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; @@ -90,12 +92,14 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import com.cloud.api.query.dao.AsyncJobJoinDao; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.dao.ImageStoreJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.dao.VolumeJoinDao; +import com.cloud.api.query.vo.AsyncJobJoinVO; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.api.query.vo.HostJoinVO; import com.cloud.api.query.vo.ImageStoreJoinVO; @@ -108,7 +112,6 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; @@ -124,12 +127,12 @@ import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.user.Account; import com.cloud.user.AccountService; -import com.cloud.user.AccountVO; import com.cloud.user.User; import com.cloud.user.UserAccount; -import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserAccountDao; import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; +import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -167,7 +170,7 @@ public class ServerAdapter extends ManagerBase { AccountService accountService; @Inject - AccountDao accountDao; + UserAccountDao userAccountDao; @Inject DataCenterDao dataCenterDao; @@ -229,6 +232,12 @@ public class ServerAdapter extends ManagerBase { @Inject NicDao nicDao; + @Inject + ApiServerService apiServerService; + + @Inject + AsyncJobJoinDao asyncJobJoinDao; + private Map jobsMap = new ConcurrentHashMap<>(); protected Role createServiceAccountRole() { @@ -255,7 +264,7 @@ public Role getServiceAccountRole() { return createServiceAccountRole(); } - protected Account createServiceAccount() { + protected UserAccount createServiceAccount() { CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); try { Role role = getServiceAccountRole(); @@ -263,23 +272,22 @@ protected Account createServiceAccount() { UUID.randomUUID().toString(), SERVICE_ACCOUNT_FIRST_NAME, SERVICE_ACCOUNT_LAST_NAME, null, null, SERVICE_ACCOUNT_NAME, Account.Type.NORMAL, role.getId(), 1L, null, null, null, null, User.Source.NATIVE); - Account account = accountService.getAccount(userAccount.getAccountId()); - logger.debug("Created Veeam service account: {}", account); - return account; + logger.debug("Created Veeam service account: {}", userAccount); + return userAccount; } finally { CallContext.unregister(); } } - protected Account createServiceAccountIfNeeded() { - List accounts = accountDao.findAccountsByName(SERVICE_ACCOUNT_NAME); - for (AccountVO account : accounts) { - if (Account.State.ENABLED.equals(account.getState())) { - logger.debug("Veeam service account found: {}", account); - return account; - } + protected Pair createServiceAccountIfNeeded() { + UserAccount userAccount = accountService.getActiveUserAccount(SERVICE_ACCOUNT_NAME, 1L); + if (userAccount == null) { + userAccount = createServiceAccount(); + } else { + logger.debug("Veeam service user account found: {}", userAccount); } - return createServiceAccount(); + return new Pair<>(accountService.getActiveUser(userAccount.getId()), + accountService.getActiveAccountById(userAccount.getAccountId())); } @Override @@ -431,8 +439,8 @@ public Vm handleCreateVm(Vm request) { bootType = ApiConstants.BootType.UEFI; bootMode = ApiConstants.BootMode.SECURE; } - Account serviceAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createVm(zoneId, clusterId, name, cpu, memory, userdata, bootType, bootMode); } finally { @@ -507,11 +515,22 @@ public VmAction startVm(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - userVmService.startVirtualMachine(vo, null); - return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); - } catch (ResourceUnavailableException | OperationTimedoutException | InsufficientCapacityException | CloudRuntimeException e) { + StartVMCmd cmd = new StartVMCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.ID, vo.getUuid()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); + return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + } catch (Exception e) { throw new CloudRuntimeException("Failed to start VM: " + e.getMessage(), e); + } finally { + CallContext.unregister(); } } @@ -520,11 +539,23 @@ public VmAction stopVm(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - userVmService.stopVirtualMachine(vo.getId(), true); - return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); - } catch (CloudRuntimeException e) { + StopVMCmd cmd = new StopVMCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.ID, vo.getUuid()); + params.put(ApiConstants.FORCED, Boolean.TRUE.toString()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); + return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + } catch (Exception e) { throw new CloudRuntimeException("Failed to stop VM: " + e.getMessage(), e); + } finally { + CallContext.unregister(); } } @@ -533,11 +564,23 @@ public VmAction shutdownVm(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - userVmService.stopVirtualMachine(vo.getId(), false); - return UserVmJoinVOToVmConverter.toVmAction(userVmJoinDao.findById(vo.getId())); - } catch (CloudRuntimeException e) { + StopVMCmd cmd = new StopVMCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.ID, vo.getUuid()); + params.put(ApiConstants.FORCED, Boolean.FALSE.toString()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); + return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + } catch (Exception e) { throw new CloudRuntimeException("Failed to shutdown VM: " + e.getMessage(), e); + } finally { + CallContext.unregister(); } } @@ -579,8 +622,8 @@ public DiskAttachment handleVmAttachDisk(final String vmUuid, final DiskAttachme if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.disk.id + " not found"); } - Account serviceAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), 0L, false); VolumeJoinVO attachedVolumeVO = volumeJoinDao.findById(volume.getId()); @@ -638,7 +681,8 @@ public Disk handleCreateDisk(Disk request) { initialSize = Long.parseLong(request.initialSize); } catch (NumberFormatException ignored) {} } - Account serviceAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + Account serviceAccount = serviceUserAccount.second(); DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { throw new InvalidParameterValueException("Datacenter for the specified storage domain is not found or not active"); @@ -647,7 +691,7 @@ public Disk handleCreateDisk(Disk request) { if (diskOfferingId == null) { throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); } - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); } finally { @@ -705,8 +749,8 @@ public Nic handleVmAttachNic(final String vmUuid, final Nic request) { if (networkVO == null) { throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().id+ " not found"); } - Account serviceAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { AddNicToVMCmd cmd = new AddNicToVMCmd(); ComponentContext.inject(cmd); @@ -775,8 +819,8 @@ public boolean handleFinalizeImageTransfer(String uuid) { } private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { - Account serviceAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceAccount.getId(), serviceAccount.getId()); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = incrementalBackupService.createImageTransfer(volumeId, null, direction, format); @@ -819,7 +863,7 @@ public List listAllJobs() { return Collections.emptyList(); } - public Job getJob(String uuid) { + public Job getTempJob(String uuid) { // final ClusterVO vo = clusterDao.findByUuid(uuid); // if (vo == null) { // throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); @@ -832,4 +876,12 @@ public Job getJob(String uuid) { return AsyncJobJoinVOToJobConverter.toJob(uuid, "started", startTime); } } + + public Job getJob(String uuid) { + final AsyncJobJoinVO vo = asyncJobJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Job with ID " + uuid + " not found"); + } + return AsyncJobJoinVOToJobConverter.toJob(vo); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index 516ea8de4d8e..5b5a62c6850d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -93,7 +93,7 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - Job response = serverAdapter.getJob(id); + Job response = serverAdapter.getTempJob(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index f3aa1dd40025..eae8ac96b11a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -19,11 +19,16 @@ import java.util.Collections; +import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.JobsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Job; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.VmAction; + +import com.cloud.api.query.vo.AsyncJobJoinVO; +import com.cloud.api.query.vo.UserVmJoinVO; public class AsyncJobJoinVOToJobConverter { @@ -47,4 +52,40 @@ public static Job toJob(String uuid, String state, long startTime) { job.setLink(Collections.emptyList()); return job; } + + public static Job toJob(AsyncJobJoinVO vo) { + Job job = new Job(); + final String basePath = VeeamControlService.ContextPath.value(); + job.setId(vo.getUuid()); + job.setHref(basePath + JobsRouteHandler.BASE_ROUTE + "/" + vo.getUuid()); + job.setAutoCleared(Boolean.TRUE.toString()); + job.setExternal(Boolean.TRUE.toString()); + job.setLastUpdated(System.currentTimeMillis()); + job.setStartTime(vo.getCreated().getTime()); + JobInfo.Status status = JobInfo.Status.values()[vo.getStatus()]; + if (status == JobInfo.Status.SUCCEEDED) { + job.setStatus("finished"); + job.setEndTime(System.currentTimeMillis()); + } else if (status == JobInfo.Status.FAILED) { + job.setStatus(status.name().toLowerCase()); + } else if (status == JobInfo.Status.CANCELLED) { + job.setStatus("aborted"); + } else { + job.setStatus("started"); + } + job.setOwner(Ref.of(basePath + "/api/users/" + vo.getUserUuid(), vo.getUserUuid())); + job.setActions(new Actions()); + job.setDescription("Something"); + job.setLink(Collections.emptyList()); + return job; + } + + public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { + VmAction action = new VmAction(); + final String basePath = VeeamControlService.ContextPath.value(); + action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null)); + action.setJob(Ref.of(basePath + JobsRouteHandler.BASE_ROUTE + vo.getUuid(), vo.getUuid())); + action.setStatus("complete"); + return action; + } } diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index dc07814c9724..1bda053ec19a 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -16,6 +16,10 @@ // under the License. package com.cloud.api; +import static com.cloud.user.AccountManagerImpl.apiKeyAccess; +import static org.apache.cloudstack.api.ApiConstants.PASSWORD_CHANGE_REQUIRED; +import static org.apache.cloudstack.user.UserPasswordResetManager.UserPasswordResetEnabled; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InterruptedIOException; @@ -31,6 +35,7 @@ import java.security.Security; import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -39,7 +44,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Arrays; import java.util.Map; import java.util.Set; import java.util.TimeZone; @@ -58,16 +62,6 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.dao.ManagementServerHostDao; -import com.cloud.utils.Ternary; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountManagerImpl; -import com.cloud.user.DomainManager; -import com.cloud.user.User; -import com.cloud.user.UserAccount; -import com.cloud.user.UserVO; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.acl.ApiKeyPairManagerImpl; import org.apache.cloudstack.acl.apikeypair.ApiKeyPair; @@ -161,6 +155,8 @@ import com.cloud.api.dispatch.DispatchChainFactory; import com.cloud.api.dispatch.DispatchTask; import com.cloud.api.response.ApiResponseSerializer; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; @@ -179,14 +175,22 @@ import com.cloud.exception.UnavailableCommandException; import com.cloud.projects.dao.ProjectDao; import com.cloud.storage.VolumeApiService; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountManagerImpl; +import com.cloud.user.DomainManager; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.UserVO; import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.DateUtil; import com.cloud.utils.HttpUtils; -import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.Pair; import com.cloud.utils.ReflectUtil; import com.cloud.utils.StringUtils; +import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -199,10 +203,6 @@ import com.cloud.utils.net.NetUtils; import com.google.gson.reflect.TypeToken; -import static com.cloud.user.AccountManagerImpl.apiKeyAccess; -import static org.apache.cloudstack.api.ApiConstants.PASSWORD_CHANGE_REQUIRED; -import static org.apache.cloudstack.user.UserPasswordResetManager.UserPasswordResetEnabled; - @Component public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiServerService, Configurable { private static final Logger ACCESSLOGGER = LogManager.getLogger("apiserver." + ApiServer.class.getName()); @@ -792,85 +792,14 @@ private String queueCommand(final BaseCmd cmdObj, final Map para // BaseAsyncCreateCmd: cmd params are processed and create() is called, then same workflow as BaseAsyncCmd. // BaseAsyncCmd: cmd is processed and submitted as an AsyncJob, job related info is serialized and returned. if (cmdObj instanceof BaseAsyncCmd) { - if (!asyncMgr.isAsyncJobsEnabled()) { - String msg = "Maintenance or Shutdown has been initiated on this management server. Can not accept new jobs"; - logger.warn(msg); - throw new ServerApiException(ApiErrorCode.SERVICE_UNAVAILABLE, msg); - } - Long objectId = null; - String objectUuid; - if (cmdObj instanceof BaseAsyncCreateCmd) { - final BaseAsyncCreateCmd createCmd = (BaseAsyncCreateCmd)cmdObj; - dispatcher.dispatchCreateCmd(createCmd, params); - objectId = createCmd.getEntityId(); - objectUuid = createCmd.getEntityUuid(); - params.put("id", objectId.toString()); - Class entityClass = EventTypes.getEntityClassForEvent(createCmd.getEventType()); - if (entityClass != null) - ctx.putContextParameter(entityClass, objectUuid); - } else { - // Extract the uuid before params are processed and id reflects internal db id - objectUuid = params.get(ApiConstants.ID); - dispatchChainFactory.getStandardDispatchChain().dispatch(new DispatchTask(cmdObj, params)); - } - - final BaseAsyncCmd asyncCmd = (BaseAsyncCmd)cmdObj; - - if (callerUserId != null) { - params.put("ctxUserId", callerUserId.toString()); - } - if (caller != null) { - params.put("ctxAccountId", String.valueOf(caller.getId())); - } - if (objectUuid != null) { - params.put("uuid", objectUuid); - } - - long startEventId = ctx.getStartEventId(); - asyncCmd.setStartEventId(startEventId); - - // save the scheduled event - final Long eventId = - ActionEventUtils.onScheduledActionEvent((callerUserId == null) ? (Long)User.UID_SYSTEM : callerUserId, asyncCmd.getEntityOwnerId(), asyncCmd.getEventType(), - asyncCmd.getEventDescription(), asyncCmd.getApiResourceId(), asyncCmd.getApiResourceType().toString(), asyncCmd.isDisplay(), startEventId); - if (startEventId == 0) { - // There was no create event before, set current event id as start eventId - startEventId = eventId; - } - - params.put("ctxStartEventId", String.valueOf(startEventId)); - params.put("cmdEventType", asyncCmd.getEventType()); - params.put("ctxDetails", ApiGsonHelper.getBuilder().create().toJson(ctx.getContextParameters())); - if (asyncCmd.getHttpMethod() != null) { - params.put(ApiConstants.HTTPMETHOD, asyncCmd.getHttpMethod().toString()); - } - - Long instanceId = (objectId == null) ? asyncCmd.getApiResourceId() : objectId; - - // users can provide the job id they want to use, so log as it is a uuid and is unique - String injectedJobId = asyncCmd.getInjectedJobId(); - uuidMgr.checkUuidSimple(injectedJobId, AsyncJob.class); - - AsyncJobVO job = new AsyncJobVO("", callerUserId, caller.getId(), cmdObj.getClass().getName(), - ApiGsonHelper.getBuilder().create().toJson(params), instanceId, - asyncCmd.getApiResourceType() != null ? asyncCmd.getApiResourceType().toString() : null, - injectedJobId); - job.setDispatcher(asyncDispatcher.getName()); - - final long jobId = asyncMgr.submitAsyncJob(job); - - if (jobId == 0L) { - final String errorMsg = "Unable to schedule async job for command " + job.getCmd(); - logger.warn(errorMsg); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); - } + AsyncCmdResult result = processAsyncCmd((BaseAsyncCmd)cmdObj, params, ctx, callerUserId, caller); final String response; - if (objectId != null) { - final String objUuid = (objectUuid == null) ? objectId.toString() : objectUuid; - response = getBaseAsyncCreateResponse(jobId, (BaseAsyncCreateCmd)asyncCmd, objUuid); + if (result.objectId != null) { + final String objUuid = (result.objectUuid == null) ? result.objectId.toString() : result.objectUuid; + response = getBaseAsyncCreateResponse(result.jobId, (BaseAsyncCreateCmd) result.asyncCmd, objUuid); } else { SerializationContext.current().setUuidTranslation(true); - response = getBaseAsyncResponse(jobId, asyncCmd); + response = getBaseAsyncResponse(result.jobId, result.asyncCmd); } // Always log response for async for now, I don't think any sensitive data will be in here. // It might be nice to send this through scrubbing similar to how @@ -900,6 +829,81 @@ private String queueCommand(final BaseCmd cmdObj, final Map para } } + @Override + public AsyncCmdResult processAsyncCmd(BaseAsyncCmd asyncCmd, Map params, CallContext ctx, Long callerUserId, Account caller) throws Exception { + if (!asyncMgr.isAsyncJobsEnabled()) { + String msg = "Maintenance or Shutdown has been initiated on this management server. Can not accept new jobs"; + logger.warn(msg); + throw new ServerApiException(ApiErrorCode.SERVICE_UNAVAILABLE, msg); + } + Long objectId = null; + String objectUuid; + if (asyncCmd instanceof BaseAsyncCreateCmd) { + final BaseAsyncCreateCmd createCmd = (BaseAsyncCreateCmd) asyncCmd; + dispatcher.dispatchCreateCmd(createCmd, params); + objectId = createCmd.getEntityId(); + objectUuid = createCmd.getEntityUuid(); + params.put("id", objectId.toString()); + Class entityClass = EventTypes.getEntityClassForEvent(createCmd.getEventType()); + if (entityClass != null) + ctx.putContextParameter(entityClass, objectUuid); + } else { + // Extract the uuid before params are processed and id reflects internal db id + objectUuid = params.get(ApiConstants.ID); + dispatchChainFactory.getStandardDispatchChain().dispatch(new DispatchTask(asyncCmd, params)); + } + + if (callerUserId != null) { + params.put("ctxUserId", callerUserId.toString()); + } + if (caller != null) { + params.put("ctxAccountId", String.valueOf(caller.getId())); + } + if (objectUuid != null) { + params.put("uuid", objectUuid); + } + + long startEventId = ctx.getStartEventId(); + asyncCmd.setStartEventId(startEventId); + + // save the scheduled event + final Long eventId = + ActionEventUtils.onScheduledActionEvent((callerUserId == null) ? (Long)User.UID_SYSTEM : callerUserId, asyncCmd.getEntityOwnerId(), asyncCmd.getEventType(), + asyncCmd.getEventDescription(), asyncCmd.getApiResourceId(), asyncCmd.getApiResourceType().toString(), asyncCmd.isDisplay(), startEventId); + if (startEventId == 0) { + // There was no create event before, set current event id as start eventId + startEventId = eventId; + } + + params.put("ctxStartEventId", String.valueOf(startEventId)); + params.put("cmdEventType", asyncCmd.getEventType()); + params.put("ctxDetails", ApiGsonHelper.getBuilder().create().toJson(ctx.getContextParameters())); + if (asyncCmd.getHttpMethod() != null) { + params.put(ApiConstants.HTTPMETHOD, asyncCmd.getHttpMethod().toString()); + } + + Long instanceId = (objectId == null) ? asyncCmd.getApiResourceId() : objectId; + + // users can provide the job id they want to use, so log as it is a uuid and is unique + String injectedJobId = asyncCmd.getInjectedJobId(); + uuidMgr.checkUuidSimple(injectedJobId, AsyncJob.class); + + AsyncJobVO job = new AsyncJobVO("", callerUserId, caller.getId(), asyncCmd.getClass().getName(), + ApiGsonHelper.getBuilder().create().toJson(params), instanceId, + asyncCmd.getApiResourceType() != null ? asyncCmd.getApiResourceType().toString() : null, + injectedJobId); + job.setDispatcher(asyncDispatcher.getName()); + + final long jobId = asyncMgr.submitAsyncJob(job); + + if (jobId == 0L) { + final String errorMsg = "Unable to schedule async job for command " + job.getCmd(); + logger.warn(errorMsg); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); + } + return new AsyncCmdResult(objectId, objectUuid, asyncCmd, jobId); + } + @SuppressWarnings("unchecked") private void buildAsyncListResponse(final BaseListCmd command, final Account account) { final List responses = ((ListResponse)command.getResponseObject()).getResponses(); From 106fbdbe308234f0f198238f6f6241d3961e5757 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 11 Feb 2026 18:22:25 +0530 Subject: [PATCH 030/129] fixes to allow worker vm deployment Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/veeam/RouteHandler.java | 7 + .../cloudstack/veeam/VeeamControlServlet.java | 54 +++--- .../veeam/adapter/ServerAdapter.java | 164 ++++++++++++++---- .../cloudstack/veeam/api/ApiService.java | 2 +- .../veeam/api/ClustersRouteHandler.java | 4 +- .../veeam/api/DataCentersRouteHandler.java | 8 +- .../veeam/api/DisksRouteHandler.java | 8 +- .../veeam/api/HostsRouteHandler.java | 4 +- .../veeam/api/ImageTransfersRouteHandler.java | 10 +- .../veeam/api/JobsRouteHandler.java | 6 +- .../veeam/api/NetworksRouteHandler.java | 4 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 162 +++++++++++++---- .../veeam/api/VnicProfilesRouteHandler.java | 4 +- .../api/converter/NicVOToNicConverter.java | 12 +- .../converter/UserVmJoinVOToVmConverter.java | 32 ++-- .../VmSnapshotVOToSnapshotConverter.java | 54 ++++++ .../VolumeJoinVOToDiskConverter.java | 12 +- .../cloudstack/veeam/api/dto/BaseDto.java | 47 +++++ .../apache/cloudstack/veeam/api/dto/Disk.java | 10 ++ .../apache/cloudstack/veeam/api/dto/Nic.java | 25 ++- .../veeam/api/dto/ReportedDevice.java | 9 + .../cloudstack/veeam/api/dto/Snapshot.java | 104 +++++++++++ .../cloudstack/veeam/api/dto/Snapshots.java | 41 +++++ .../apache/cloudstack/veeam/api/dto/Vm.java | 12 +- .../veeam/api/dto/VmInitialization.java | 10 +- .../services/PkiResourceRouteHandler.java | 2 +- .../cloudstack/veeam/sso/SsoService.java | 2 +- 27 files changed, 651 insertions(+), 158 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index fa7ab174f2b7..a955eeac0203 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -19,6 +19,7 @@ import java.io.BufferedReader; import java.io.IOException; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -64,4 +65,10 @@ static String getRequestData(HttpServletRequest req) { return null; } } + + static Map getRequestParams(HttpServletRequest req) { + return req.getParameterMap().entrySet().stream() + .filter(e -> e.getValue() != null && e.getValue().length > 0) + .collect(java.util.stream.Collectors.toMap(Map.Entry::getKey, e -> e.getValue()[0])); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java index 69f6b9fb5c06..8016bf9c17a4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -27,8 +27,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.Mapper; +import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.ResponseWriter; import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -36,6 +36,7 @@ public class VeeamControlServlet extends HttpServlet { private static final Logger LOGGER = LogManager.getLogger(VeeamControlServlet.class); + private static final boolean LOG_REQUESTS = false; private final ResponseWriter writer; private final Mapper mapper; @@ -63,6 +64,32 @@ protected void service(HttpServletRequest req, HttpServletResponse resp) throws LOGGER.info("Received {} request for {} with out format: {}", method, path, outFormat); + logRequest(req, method, path); + + try { + if ("/".equals(path)) { + handleRoot(req, resp, outFormat); + return; + } + + if (CollectionUtils.isNotEmpty(this.routeHandlers)) { + for (RouteHandler handler : this.routeHandlers) { + if (handler.canHandle(method, path)) { + handler.handle(req, resp, path, outFormat, this); + return; + } + } + } + notFound(resp, null, outFormat); + } catch (Error e) { + writer.writeFault(resp, e.status, e.message, null, outFormat); + } + } + + private static void logRequest(HttpServletRequest req, String method, String path) { + if (!LOG_REQUESTS) { + return; + } // Add a log to give all info about the request try { StringBuilder details = new StringBuilder(); @@ -91,25 +118,6 @@ protected void service(HttpServletRequest req, HttpServletResponse resp) throws } catch (Exception e) { LOGGER.debug("Failed to capture request details", e); } - - try { - if ("/".equals(path)) { - handleRoot(req, resp, outFormat); - return; - } - - if (CollectionUtils.isNotEmpty(this.routeHandlers)) { - for (RouteHandler handler : this.routeHandlers) { - if (handler.canHandle(method, path)) { - handler.handle(req, resp, path, outFormat, this); - return; - } - } - } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); - } catch (Error e) { - writer.writeFault(resp, e.status, e.message, null, outFormat); - } } private String normalize(String pathInfo) { @@ -133,16 +141,16 @@ protected void handleRoot(HttpServletRequest req, HttpServletResponse resp, Nego public void methodNotAllowed(final HttpServletResponse resp, final String allow, final Negotiation.OutFormat outFormat) throws IOException { resp.setHeader("Allow", allow); - writer.writeFault(resp, 405, "Method Not Allowed", "Allowed methods: " + allow, outFormat); + writer.writeFault(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, "Method Not Allowed", "Allowed methods: " + allow, outFormat); } public void badRequest(final HttpServletResponse resp, String detail, Negotiation.OutFormat outFormat) throws IOException { - writer.writeFault(resp, 400, "Bad request", detail, outFormat); + writer.writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", detail, outFormat); } public void notFound(final HttpServletResponse resp, String detail, Negotiation.OutFormat outFormat) throws IOException { - writer.writeFault(resp, 404, "Not found", detail, outFormat); + writer.writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", detail, outFormat); } public static class Error extends RuntimeException { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index b7d8e2699766..468d329b07bd 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.veeam.adapter; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -24,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import javax.inject.Inject; @@ -46,6 +46,8 @@ import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; import org.apache.cloudstack.api.command.user.vm.StopVMCmd; +import org.apache.cloudstack.api.command.user.vmsnapshot.CreateVMSnapshotCmd; +import org.apache.cloudstack.api.command.user.vmsnapshot.DeleteVMSnapshotCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; @@ -73,6 +75,7 @@ import org.apache.cloudstack.veeam.api.converter.NicVOToNicConverter; import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; +import org.apache.cloudstack.veeam.api.converter.VmSnapshotVOToSnapshotConverter; import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.DataCenter; @@ -84,6 +87,7 @@ import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.StorageDomain; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.dto.VmAction; @@ -139,8 +143,12 @@ import com.cloud.vm.NicVO; import com.cloud.vm.UserVmService; import com.cloud.vm.UserVmVO; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.snapshot.VMSnapshotService; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; public class ServerAdapter extends ManagerBase { private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; @@ -162,6 +170,7 @@ public class ServerAdapter extends ManagerBase { ResizeVolumeCmd.class, ListNetworksCmd.class ); + public static final String GUEST_CPU_MODE = "host-passthrough"; @Inject RoleService roleService; @@ -238,7 +247,13 @@ public class ServerAdapter extends ManagerBase { @Inject AsyncJobJoinDao asyncJobJoinDao; - private Map jobsMap = new ConcurrentHashMap<>(); + @Inject + VMSnapshotDao vmSnapshotDao; + + @Inject + VMSnapshotService vmSnapshotService; + + //ToDo: check access on objects protected Role createServiceAccountRole() { Role role = roleService.createRole(SERVICE_ACCOUNT_ROLE_NAME, RoleType.User, @@ -383,13 +398,13 @@ public VnicProfile getVnicProfile(String uuid) { return NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); } - public List listAllUserVms() { + public List listAllInstances() { // Todo: add filtering, pagination List vms = userVmJoinDao.listAll(); return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById); } - public Vm getVm(String uuid) { + public Vm getInstance(String uuid) { UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -398,7 +413,7 @@ public Vm getVm(String uuid) { this::listNicsByInstance); } - public Vm handleCreateVm(Vm request) { + public Vm createInstance(Vm request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); } @@ -424,14 +439,14 @@ public Vm handleCreateVm(Vm request) { } Long memory = null; try { - memory = request.memory; + memory = Long.valueOf(request.memory); } catch (Exception ignored) {} if (memory == null) { throw new InvalidParameterValueException("Memory must be specified"); } String userdata = null; if (request.getInitialization() != null) { - userdata = request.getInitialization().getContentData(); + userdata = request.getInitialization().getCustomScript(); } ApiConstants.BootType bootType = ApiConstants.BootType.BIOS; ApiConstants.BootMode bootMode = ApiConstants.BootMode.LEGACY; @@ -442,7 +457,7 @@ public Vm handleCreateVm(Vm request) { Pair serviceUserAccount = createServiceAccountIfNeeded(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - return createVm(zoneId, clusterId, name, cpu, memory, userdata, bootType, bootMode); + return createInstance(zoneId, clusterId, name, cpu, memory, userdata, bootType, bootMode); } finally { CallContext.unregister(); } @@ -463,20 +478,21 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(long zoneId, int cpu return serviceOfferingDao.findByUuid(uuid); } - protected Vm createVm(Long zoneId, Long clusterId, String name, int cpu, long memory, String userdata, - ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { + protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, long memory, String userdata, + ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zoneId, cpu, memory); if (serviceOffering == null) { throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); } DeployVMCmdByAdmin cmd = new DeployVMCmdByAdmin(); + cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); cmd.setZoneId(zoneId); cmd.setClusterId(clusterId); cmd.setName(name); cmd.setServiceOfferingId(serviceOffering.getId()); if (StringUtils.isNotEmpty(userdata)) { - cmd.setUserData(Base64.getEncoder().encodeToString(userdata.getBytes())); + cmd.setUserData(Base64.getEncoder().encodeToString(userdata.getBytes(StandardCharsets.UTF_8))); } if (bootType != null) { cmd.setBootType(bootType.toString()); @@ -487,6 +503,11 @@ protected Vm createVm(Long zoneId, Long clusterId, String name, int cpu, long me // ToDo: handle other. cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); cmd.setBlankInstance(true); + Map details = new HashMap<>(); + details.put(VmDetailConstants.GUEST_CPU_MODE, GUEST_CPU_MODE); + Map> map = new HashMap<>(); + map.put(0, details); + cmd.setDetails(map); try { UserVm vm = userVmService.createVirtualMachine(cmd); vm = userVmService.finalizeCreateVirtualMachine(vm.getId()); @@ -498,7 +519,11 @@ protected Vm createVm(Long zoneId, Long clusterId, String name, int cpu, long me } } - public void deleteVm(String uuid) { + public Vm updateInstance(String uuid, Vm request) { + return getInstance(uuid); + } + + public void deleteInstance(String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -510,7 +535,7 @@ public void deleteVm(String uuid) { } } - public VmAction startVm(String uuid) { + public VmAction startInstance(String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -519,6 +544,7 @@ public VmAction startVm(String uuid) { CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartVMCmd cmd = new StartVMCmd(); + cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); @@ -534,7 +560,7 @@ public VmAction startVm(String uuid) { } } - public VmAction stopVm(String uuid) { + public VmAction stopInstance(String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -543,6 +569,7 @@ public VmAction stopVm(String uuid) { CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); + cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); @@ -559,7 +586,7 @@ public VmAction stopVm(String uuid) { } } - public VmAction shutdownVm(String uuid) { + public VmAction shutdownInstance(String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -568,6 +595,7 @@ public VmAction shutdownVm(String uuid) { CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); + cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); @@ -610,7 +638,7 @@ public List listDiskAttachmentsByInstanceUuid(final String uuid) return listDiskAttachmentsByInstanceId(vo.getId()); } - public DiskAttachment handleVmAttachDisk(final String vmUuid, final DiskAttachment request) { + public DiskAttachment handleInstanceAttachDisk(final String vmUuid, final DiskAttachment request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); @@ -729,7 +757,7 @@ protected List listNicsByInstance(final UserVmJoinVO vo) { return listNicsByInstance(vo.getId(), vo.getUuid()); } - public List listNicsByInstanceId(final String uuid) { + public List listNicsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -737,7 +765,7 @@ public List listNicsByInstanceId(final String uuid) { return listNicsByInstance(vo.getId(), vo.getUuid()); } - public Nic handleVmAttachNic(final String vmUuid, final Nic request) { + public Nic handleAttachInstanceNic(final String vmUuid, final Nic request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); @@ -860,28 +888,98 @@ protected NetworkVO getNetworkById(Long networkId) { } public List listAllJobs() { + // ToDo: find active jobs for service account return Collections.emptyList(); } - public Job getTempJob(String uuid) { -// final ClusterVO vo = clusterDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); -// } - long startTime = jobsMap.computeIfAbsent(uuid, k -> System.currentTimeMillis()); - long elapsed = System.currentTimeMillis() - startTime; - if (elapsed > 10000L) { - return AsyncJobJoinVOToJobConverter.toJob(uuid, "finished", startTime); - } else { - return AsyncJobJoinVOToJobConverter.toJob(uuid, "started", startTime); - } - } - public Job getJob(String uuid) { - final AsyncJobJoinVO vo = asyncJobJoinDao.findByUuid(uuid); + final AsyncJobJoinVO vo = asyncJobJoinDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Job with ID " + uuid + " not found"); } return AsyncJobJoinVOToJobConverter.toJob(vo); } + + public List listSnapshotsByInstanceUuid(final String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + List snapshots = vmSnapshotDao.findByVm(vo.getId()); + return VmSnapshotVOToSnapshotConverter.toSnapshotList(snapshots, vo.getUuid()); + } + + public Snapshot handleCreateInstanceSnapshot(final String vmUuid, final Snapshot request) { + UserVmVO vmVo = userVmDao.findByUuid(vmUuid); + if (vmVo == null) { + throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); + } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + CreateVMSnapshotCmd cmd = new CreateVMSnapshotCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); + params.put(ApiConstants.VM_SNAPSHOT_DESCRIPTION, request.getDescription()); + params.put(ApiConstants.VM_SNAPSHOT_MEMORY, String.valueOf(Boolean.parseBoolean(request.getPersistMemorystate()))); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + if (result.objectId == null) { + throw new CloudRuntimeException("No snapshot ID returned"); + } + VMSnapshotVO vo = vmSnapshotDao.findById(result.objectId); + if (vo == null) { + throw new CloudRuntimeException("Snapshot not found"); + } + return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vmVo.getUuid()); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to create snapshot: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } + } + + public Snapshot getSnapshot(String uuid) { + VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); + } + UserVmVO vm = userVmDao.findById(vo.getVmId()); + return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); + } + + public Snapshot deleteSnapshot(String uuid, boolean async) { + Snapshot snapshot = null; + VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); + } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + if (async) { + DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.ID, vo.getUuid()); + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + vo = vmSnapshotDao.findById(vo.getId()); + if (vo == null) { + throw new CloudRuntimeException("Snapshot not found"); + } + UserVmVO vm = userVmDao.findById(vo.getVmId()); + snapshot = VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); + } else { + vmSnapshotService.deleteVMSnapshot(vo.getId()); + } + } catch (Exception e) { + throw new CloudRuntimeException("Failed to delete snapshot: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } + return snapshot; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index 380a64715fea..dd0e4b250826 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -61,7 +61,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleRootApiRequest(req, resp, outFormat, io); return; } - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", null, outFormat); + io.notFound(resp, null, outFormat); } private void handleRootApiRequest(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index a80d0ec8d611..37ef228db9f3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -79,7 +79,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -96,7 +96,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi Cluster response = serverAdapter.getCluster(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index e2e60fe8479e..dd324eb9ee3e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -93,7 +93,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -110,7 +110,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi DataCenter response = serverAdapter.getDataCenter(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -121,7 +121,7 @@ protected void handleGetStorageDomainsByDcId(final String id, final HttpServletR StorageDomains response = new StorageDomains(storageDomains); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -132,7 +132,7 @@ protected void handleGetNetworksByDcId(final String id, final HttpServletRespons Networks response = new Networks(networks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 0bd618a8111e..fa1248539b1b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -93,7 +93,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -113,7 +113,7 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons Disk response = serverAdapter.handleCreateDisk(request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -123,7 +123,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi Disk response = serverAdapter.getDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -133,7 +133,7 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, serverAdapter.deleteDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Deleted disk ID: " + id, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java index 37ac17b23642..efe41bfbe301 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -79,7 +79,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -96,7 +96,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi Host response = serverAdapter.getHost(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 3cdd5d0469d8..9c77a28e426f 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -100,7 +100,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -120,7 +120,7 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons ImageTransfer response = serverAdapter.handleCreateImageTransfer(request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad Request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -130,7 +130,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi ImageTransfer response = serverAdapter.getImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -140,7 +140,7 @@ protected void handleCancelById(final String id, final HttpServletResponse resp, serverAdapter.handleCancelImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer cancelled successfully", outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -150,7 +150,7 @@ protected void handleFinalizeById(final String id, final HttpServletResponse res serverAdapter.handleFinalizeImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer finalized successfully", outFormat); } catch (CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index 5b5a62c6850d..7213cdac5bed 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -79,7 +79,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -93,10 +93,10 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - Job response = serverAdapter.getTempJob(id); + Job response = serverAdapter.getJob(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index d11397e1eee5..2450c85cf517 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -79,7 +79,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -96,7 +96,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi Network response = serverAdapter.getNetwork(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 30d781e868b6..103b33b3c6ab 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -32,6 +32,8 @@ import org.apache.cloudstack.veeam.api.dto.DiskAttachments; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Nics; +import org.apache.cloudstack.veeam.api.dto.Snapshot; +import org.apache.cloudstack.veeam.api.dto.Snapshots; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.dto.VmAction; import org.apache.cloudstack.veeam.api.dto.Vms; @@ -105,7 +107,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.methodNotAllowed(resp, "GET, PUT, DELETE", outFormat); } else if ("GET".equalsIgnoreCase(method)) { handleGetById(id, resp, outFormat, io); - } else if ("DELETE".equalsIgnoreCase(method)) { + } else if ("PUT".equalsIgnoreCase(method)) { handleUpdateById(id, req, resp, outFormat, io); } else if ("DELETE".equalsIgnoreCase(method)) { handleDeleteById(id, resp, outFormat, io); @@ -152,11 +154,51 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handlePostNicForVmId(id, req, resp, outFormat, io); } return; + } else if ("snapshots".equals(subPath)) { + if (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, POST", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetSnapshotsByVmId(id, resp, outFormat, io); + } else if ("POST".equalsIgnoreCase(method)) { + handlePostSnapshotForVmId(id, req, resp, outFormat, io); + } + return; + } + } else if (idAndSubPath.size() == 3) { + String subPath = idAndSubPath.get(1); + String subId = idAndSubPath.get(2); + if ("snapshots".equals(subPath)) { + if (!"GET".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, DELETE", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetSnapshotsById(subId, resp, outFormat, io); + } else if ("DELETE".equalsIgnoreCase(method)) { + handleDeleteSnapshotById(subId, req, resp, outFormat, io); + } + return; + } + } else if (idAndSubPath.size() == 4) { + String subPath = idAndSubPath.get(1); + String subId = idAndSubPath.get(2); + String action = idAndSubPath.get(3); + if ("snapshots".equals(subPath) && "restore".equals(action)) { + if ("POST".equalsIgnoreCase(method)) { + handleRestoreSnapshotById(subId, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; } } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); + } + + protected String getRequestData(final HttpServletRequest req) { + String data = RouteHandler.getRequestData(req); + logger.info("Received method: {} request. Request-data: {}", req.getMethod(), data); + return data; } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -192,7 +234,7 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse return; } - final List result = serverAdapter.listAllUserVms(); + final List result = serverAdapter.listAllInstances(); final Vms response = new Vms(result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); @@ -217,71 +259,76 @@ protected static Integer parseIntOrNull(final String s) { protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); - logger.info("Received method: POST request. Request-data: {}", data); + String data = getRequestData(req); try { Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); - Vm response = serverAdapter.handleCreateVm(request); + Vm response = serverAdapter.createInstance(request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - Vm response = serverAdapter.getVm(id); + Vm response = serverAdapter.getInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handleUpdateById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req); - logger.info("Received POST request, but method: POST is not supported atm. Request-data: {}", data); - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Not implemented", "", outFormat); + logger.info("Received PUT request. Request-data: {}", data); + try { + Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); + Vm response = serverAdapter.updateInstance(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } } protected void handleDeleteById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - serverAdapter.deleteVm(id); + serverAdapter.deleteInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "", outFormat); } catch (CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - VmAction vm = serverAdapter.startVm(id); + VmAction vm = serverAdapter.startInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - VmAction vm = serverAdapter.stopVm(id); + VmAction vm = serverAdapter.stopInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - VmAction vm = serverAdapter.shutdownVm(id); + VmAction vm = serverAdapter.shutdownInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } @@ -292,46 +339,101 @@ protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServlet DiskAttachments response = new DiskAttachments(disks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handlePostDiskAttachmentForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); - logger.info("Received method: POST request. Request-data: {}", data); + String data = getRequestData(req); try { DiskAttachment request = io.getMapper().jsonMapper().readValue(data, DiskAttachment.class); - DiskAttachment response = serverAdapter.handleVmAttachDisk(id, request); + DiskAttachment response = serverAdapter.handleInstanceAttachDisk(id, request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } protected void handleGetNicsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - List nics = serverAdapter.listNicsByInstanceId(id); + List nics = serverAdapter.listNicsByInstanceUuid(id); Nics response = new Nics(nics); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } protected void handlePostNicForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); - logger.info("Received method: POST request. Request-data: {}", data); + String data = getRequestData(req); try { Nic request = io.getMapper().jsonMapper().readValue(data, Nic.class); - Nic response = serverAdapter.handleVmAttachNic(id, request); + Nic response = serverAdapter.handleAttachInstanceNic(id, request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_BAD_REQUEST, "Bad request", e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handleGetSnapshotsByVmId(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + List snapshots = serverAdapter.listSnapshotsByInstanceUuid(id); + Snapshots response = new Snapshots(snapshots); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); } } + + protected void handlePostSnapshotForVmId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = getRequestData(req); + try { + Snapshot request = io.getMapper().jsonMapper().readValue(data, Snapshot.class); + Snapshot response = serverAdapter.handleCreateInstanceSnapshot(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handleGetSnapshotsById(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + Snapshot response = serverAdapter.getSnapshot(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } + + protected void handleDeleteSnapshotById(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + boolean async = Boolean.parseBoolean(req.getParameter("async")); + try { + Snapshot snapshot = serverAdapter.deleteSnapshot(id, async); + if (snapshot != null) { + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, null, outFormat); + } else { + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, null, outFormat); + } + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handleRestoreSnapshotById(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = getRequestData(req); + io.badRequest(resp, "Not implemented", outFormat); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index c62fbf694821..a0ce779d6446 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -79,7 +79,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, @@ -96,7 +96,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi VnicProfile response = serverAdapter.getVnicProfile(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.getWriter().writeFault(resp, HttpServletResponse.SC_NOT_FOUND, "Not found", e.getMessage(), outFormat); + io.notFound(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java index 72fe2d559656..204844649ae4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.VnicProfilesRouteHandler; import org.apache.cloudstack.veeam.api.dto.Ip; import org.apache.cloudstack.veeam.api.dto.Ips; @@ -46,10 +47,11 @@ public static Nic toNic(final NicVO vo, final String vmUuid, final Function h dst.description = src.getDisplayName(); dst.href = basePath + VmsRouteHandler.BASE_ROUTE + "/" + src.getUuid(); dst.status = mapStatus(src.getState()); + dst.setCreationTime(src.getCreated().getTime()); final Date lastUpdated = src.getLastUpdated() != null ? src.getLastUpdated() : src.getCreated(); if ("down".equals(dst.status)) { dst.stopTime = lastUpdated.getTime(); @@ -106,7 +107,7 @@ public static Vm toVm(final UserVmJoinVO src, final Function h } } - dst.memory = src.getRamSize() * 1024L * 1024L; + dst.memory = String.valueOf(src.getRamSize() * 1024L * 1024L); dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); dst.os = new Os(); @@ -129,17 +130,15 @@ public static Vm toVm(final UserVmJoinVO src, final Function h } dst.actions = new Actions(List.of( - new Link("start", dst.href + "/start"), - new Link("stop", dst.href + "/stop"), - new Link("shutdown", dst.href + "/shutdown") + BaseDto.getActionLink("start", dst.href), + BaseDto.getActionLink("stop", dst.href), + BaseDto.getActionLink("shutdown", dst.href) )); dst.link = List.of( - new Link("diskattachments", - dst.href + "/diskattachments"), - new Link("nics", - dst.href + "/nics"), - new Link("snapshots", - dst.href + "/snapshots") + BaseDto.getActionLink("diskattachments", dst.href), + BaseDto.getActionLink("nics", dst.href), + BaseDto.getActionLink("reporteddevices", dst.href), + BaseDto.getActionLink("snapshots", dst.href) ); dst.tags = new EmptyElement(); @@ -162,21 +161,12 @@ public static VmAction toVmAction(final UserVmJoinVO vm) { } private static String mapStatus(final VirtualMachine.State state) { - if (state == null) { - return null; - } - // CloudStack-ish states -> oVirt-ish up/down if (Arrays.asList(VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Migrating, VirtualMachine.State.Restoring).contains(state)) { return "up"; } - if (Arrays.asList(VirtualMachine.State.Stopped, VirtualMachine.State.Stopping, - VirtualMachine.State.Shutdown, VirtualMachine.State.Error, - VirtualMachine.State.Expunging).contains(state)) { - return "down"; - } - return null; + return "down"; } private static Ref buildRef(final String baseHref, final String suffix, final String id) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java new file mode 100644 index 000000000000..cf7226227b0c --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Actions; +import org.apache.cloudstack.veeam.api.dto.BaseDto; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Snapshot; + +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; + +public class VmSnapshotVOToSnapshotConverter { + public static Snapshot toSnapshot(final VMSnapshotVO vmSnapshotVO, String vmUuid) { + final String basePath = VeeamControlService.ContextPath.value(); + final Snapshot snapshot = new Snapshot(); + snapshot.setId(vmSnapshotVO.getUuid()); + snapshot.setHref(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmUuid + "/snapshots/" + vmSnapshotVO.getUuid()); + snapshot.setVm(Ref.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmUuid, vmUuid)); + snapshot.setDescription(vmSnapshotVO.getDescription()); + snapshot.setSnapshotType("active"); + snapshot.setDate(vmSnapshotVO.getCreated().getTime()); + snapshot.setPersistMemorystate(String.valueOf(VMSnapshotVO.Type.DiskAndMemory.equals(vmSnapshotVO.getType()))); + snapshot.setSnapshotStatus(VMSnapshot.State.Ready.equals(vmSnapshotVO.getState()) ? "ok" : "locked"); + snapshot.setActions(new Actions(List.of(BaseDto.getActionLink("restore", snapshot.getHref())))); + return snapshot; + } + + public static List toSnapshotList(final List vmSnapshotVOList, final String vmUuid) { + return vmSnapshotVOList.stream() + .map(v -> toSnapshot(v, vmUuid)) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 0bb8e40d92a3..015b0076334c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -40,13 +40,14 @@ public class VolumeJoinVOToDiskConverter { public static Disk toDisk(final VolumeJoinVO vol) { final Disk disk = new Disk(); - final String basePath = VeeamControlService.ContextPath.value() + ApiService.BASE_ROUTE; - + final String basePath = VeeamControlService.ContextPath.value(); + final String apiBasePath = basePath + ApiService.BASE_ROUTE; final String diskId = vol.getUuid(); final String diskHref = basePath + DisksRouteHandler.BASE_ROUTE + "/" + diskId; disk.id = diskId; disk.href = diskHref; + disk.setBootable(String.valueOf(Volume.Type.ROOT.equals(vol.getVolumeType()))); // Names disk.name = vol.getName(); @@ -98,7 +99,7 @@ public static Disk toDisk(final VolumeJoinVO vol) { // Disk profile (optional) disk.diskProfile = Ref.of( - basePath + "/diskprofiles/" + vol.getDiskOfferingUuid(), + apiBasePath + "/diskprofiles/" + vol.getDiskOfferingUuid(), String.valueOf(vol.getDiskOfferingUuid()) ); @@ -107,7 +108,7 @@ public static Disk toDisk(final VolumeJoinVO vol) { Disk.StorageDomains sds = new Disk.StorageDomains(); sds.storageDomain = List.of( Ref.of( - basePath + "/storagedomains/" + vol.getPoolUuid(), + apiBasePath + "/storagedomains/" + vol.getPoolUuid(), vol.getPoolUuid() ) ); @@ -134,7 +135,6 @@ public static List toDiskList(final List srcList) { public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { final DiskAttachment da = new DiskAttachment(); final String basePath = VeeamControlService.ContextPath.value(); - final String apiBase = basePath + ApiService.BASE_ROUTE; final String diskAttachmentId = vol.getUuid(); da.vm = Ref.of( @@ -143,7 +143,7 @@ public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { ); da.id = diskAttachmentId; - da.href = da.vm.href + "/diskattachements/" + diskAttachmentId;; + da.href = da.vm.href + "/diskattachments/" + diskAttachmentId;; // Links da.disk = toDisk(vol); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java new file mode 100644 index 000000000000..013dd9145d90 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class BaseDto { + + private String href; + private String id; + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public static Link getActionLink(final String action, final String baseHref) { + return new Link(action, baseHref + "/" + action); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java index f61cd5d890e3..6ba2f1d736b8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java @@ -28,6 +28,8 @@ @JacksonXmlRootElement(localName = "disk") public final class Disk { + private String bootable; + @JsonProperty("actual_size") public String actualSize; @@ -88,6 +90,14 @@ public final class Disk { public Disk() {} + public String getBootable() { + return bootable; + } + + public void setBootable(String bootable) { + this.bootable = bootable; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlRootElement(localName = "storage_domains") public static final class StorageDomains { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java index 7eca9aff4f78..dcb9d3505a38 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java @@ -34,6 +34,7 @@ public class Nic { private String linked; private Mac mac; private String plugged; + public String synced; private Ref vnicProfile; private Ref vm; private ReportedDevices reportedDevices; @@ -81,12 +82,12 @@ public void setInterfaceType(String interfaceType) { this.interfaceType = interfaceType; } - public boolean isLinked() { - return Boolean.parseBoolean(linked); + public String getLinked() { + return linked; } - public void setLinked(boolean linked) { - this.linked = Boolean.toString(linked); + public void setLinked(String linked) { + this.linked = linked; } public Mac getMac() { @@ -97,12 +98,20 @@ public void setMac(Mac mac) { this.mac = mac; } - public boolean isPlugged() { - return Boolean.parseBoolean(plugged); + public String getPlugged() { + return plugged; } - public void setPlugged(boolean plugged) { - this.plugged = Boolean.toString(plugged); + public void setPlugged(String plugged) { + this.plugged = plugged; + } + + public String getSynced() { + return synced; + } + + public void setSynced(String synced) { + this.synced = synced; } public Ref getVnicProfile() { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java index 7c36f2d02f5a..14a540699bb0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java @@ -25,6 +25,7 @@ public class ReportedDevice { private Mac Mac; private String name; private String type; + private String href; private Ref vm; public String getComment() { @@ -83,6 +84,14 @@ public void setType(String type) { this.type = type; } + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + public Ref getVm() { return vm; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java new file mode 100644 index 000000000000..5f5347e1181d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Snapshot extends BaseDto { + + // epoch millis + private Long date; + private String persistMemorystate; + private String snapshotStatus; + private String snapshotType; + private Actions actions; + private String description; + @JacksonXmlElementWrapper(useWrapping = false) + private List link; + private Ref vm; + + public Snapshot() {} + + public Long getDate() { + return date; + } + + public void setDate(final Long date) { + this.date = date; + } + + public String getPersistMemorystate() { + return persistMemorystate; + } + + public void setPersistMemorystate(final String persistMemorystate) { + this.persistMemorystate = persistMemorystate; + } + + public String getSnapshotStatus() { + return snapshotStatus; + } + + public void setSnapshotStatus(final String snapshotStatus) { + this.snapshotStatus = snapshotStatus; + } + + public String getSnapshotType() { + return snapshotType; + } + + public void setSnapshotType(final String snapshotType) { + this.snapshotType = snapshotType; + } + + public Actions getActions() { + return actions; + } + + public void setActions(final Actions actions) { + this.actions = actions; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public List getLink() { + return link; + } + + public void setLink(final List link) { + this.link = link; + } + + public Ref getVm() { + return vm; + } + + public void setVm(Ref vm) { + this.vm = vm; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java new file mode 100644 index 000000000000..66a9b93e46d6 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JacksonXmlRootElement(localName = "snapshots") +public final class Snapshots { + + @JsonProperty("snapshot") + @JacksonXmlElementWrapper(useWrapping = false) + public List snapshot; + + public Snapshots() {} + + public Snapshots(final List snapshot) { + this.snapshot = snapshot; + } +} + diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index c83a7536e6a2..2438109105fc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -43,6 +43,8 @@ public final class Vm { @JacksonXmlProperty(localName = "stop_reason") public String stopReason; // empty string allowed + private Long creationTime; + @JsonProperty("stop_time") @JacksonXmlProperty(localName = "stop_time") public Long stopTime; // epoch millis @@ -57,7 +59,7 @@ public final class Vm { public Ref cluster; public Ref host; - public Long memory; // bytes + public String memory; // bytes public Cpu cpu; public Os os; public Bios bios; @@ -77,6 +79,14 @@ public final class Vm { public Vm() {} + public Long getCreationTime() { + return creationTime; + } + + public void setCreationTime(Long creationTime) { + this.creationTime = creationTime; + } + public Long getStartTime() { return startTime; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java index 61982872afca..a9e77b01a1cc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java @@ -22,13 +22,13 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class VmInitialization { - private String contentData; + private String customScript; - public String getContentData() { - return contentData; + public String getCustomScript() { + return customScript; } - public void setContentData(String contentData) { - this.contentData = contentData; + public void setCustomScript(String customScript) { + this.customScript = customScript; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java index 19b1b88d7f34..0e2037ba9db0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java @@ -65,7 +65,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleGet(HttpServletRequest req, HttpServletResponse resp, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java index c80668239990..26a29d6d5317 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -55,7 +55,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } - resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + io.notFound(resp, null, outFormat); } protected void handleToken(HttpServletRequest req, HttpServletResponse resp, From b97f70c116929200bdc7d9ea3692ef6d14817a4d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 11 Feb 2026 18:23:18 +0530 Subject: [PATCH 031/129] userdata: defensive check for userdata validation Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/userdata/UserDataManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java index 7c5692564c99..c9c48dbb179f 100644 --- a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java +++ b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java @@ -119,10 +119,10 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) { byte[] decodedUserData = null; // If GET, use 4K. If POST, support up to 1M. - if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); - } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { + if (BaseCmd.HTTPMethod.POST.equals(httpmethod)) { decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); + } else { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); } // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. From 047595d938964c214b4319688a477ebbabefd81f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 12 Feb 2026 01:25:42 +0530 Subject: [PATCH 032/129] fix snapshot delete Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 23 +++++------ .../cloudstack/veeam/api/VmsRouteHandler.java | 12 +++--- .../AsyncJobJoinVOToJobConverter.java | 17 ++++++-- .../veeam/api/dto/ResourceAction.java | 39 +++++++++++++++++++ .../cloudstack/veeam/api/dto/VmAction.java | 23 +---------- 5 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ResourceAction.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 468d329b07bd..c5bb2c60fd0d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -87,6 +87,7 @@ import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.StorageDomain; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -950,8 +951,8 @@ public Snapshot getSnapshot(String uuid) { return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); } - public Snapshot deleteSnapshot(String uuid, boolean async) { - Snapshot snapshot = null; + public ResourceAction deleteSnapshot(String uuid, boolean async) { + ResourceAction action = null; VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); @@ -963,15 +964,15 @@ public Snapshot deleteSnapshot(String uuid, boolean async) { DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); ComponentContext.inject(cmd); Map params = new HashMap<>(); - params.put(ApiConstants.ID, vo.getUuid()); - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); - vo = vmSnapshotDao.findById(vo.getId()); - if (vo == null) { - throw new CloudRuntimeException("Snapshot not found"); + params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for snapshot deletion"); } - UserVmVO vm = userVmDao.findById(vo.getVmId()); - snapshot = VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); + action = AsyncJobJoinVOToJobConverter.toAction(jobVo); } else { vmSnapshotService.deleteVMSnapshot(vo.getId()); } @@ -980,6 +981,6 @@ public Snapshot deleteSnapshot(String uuid, boolean async) { } finally { CallContext.unregister(); } - return snapshot; + return action; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 103b33b3c6ab..908aece8bdfb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.veeam.api.dto.DiskAttachments; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Nics; +import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.Snapshots; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -417,13 +418,14 @@ protected void handleGetSnapshotsById(final String id, final HttpServletResponse protected void handleDeleteSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = Boolean.parseBoolean(req.getParameter("async")); + String asyncStr = req.getParameter("async"); + boolean async = !Boolean.FALSE.toString().equals(asyncStr); try { - Snapshot snapshot = serverAdapter.deleteSnapshot(id, async); - if (snapshot != null) { - io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, null, outFormat); + ResourceAction action = serverAdapter.deleteSnapshot(id, async); + if (action != null) { + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, action, outFormat); } else { - io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, null, outFormat); + io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); } } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index eae8ac96b11a..6c273a22f286 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Job; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.VmAction; import com.cloud.api.query.vo.AsyncJobJoinVO; @@ -80,12 +81,22 @@ public static Job toJob(AsyncJobJoinVO vo) { return job; } - public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { - VmAction action = new VmAction(); + protected static void fillAction(final ResourceAction action, final AsyncJobJoinVO vo) { final String basePath = VeeamControlService.ContextPath.value(); - action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null)); action.setJob(Ref.of(basePath + JobsRouteHandler.BASE_ROUTE + vo.getUuid(), vo.getUuid())); action.setStatus("complete"); + } + + public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { + VmAction action = new VmAction(); + fillAction(action, vo); + action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null)); + return action; + } + + public static ResourceAction toAction(final AsyncJobJoinVO vo) { + VmAction action = new VmAction(); + fillAction(action, vo); return action; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ResourceAction.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ResourceAction.java new file mode 100644 index 000000000000..ed6c39240369 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ResourceAction.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +public class ResourceAction extends BaseDto { + private Ref job; + private String status; + + public Ref getJob() { + return job; + } + + public void setJob(Ref job) { + this.job = job; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java index 9be7ab6891e5..2fb5d11d0789 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java @@ -17,21 +17,8 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class VmAction { - private Ref job; +public class VmAction extends ResourceAction { private Vm vm; - private String status; - - public Ref getJob() { - return job; - } - - public void setJob(Ref job) { - this.job = job; - } public Vm getVm() { return vm; @@ -40,12 +27,4 @@ public Vm getVm() { public void setVm(Vm vm) { this.vm = vm; } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } } From 2352c83378b7df3e096be64fd09f8836ab58f67d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 12 Feb 2026 11:03:41 +0530 Subject: [PATCH 033/129] return job for async=false as well Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index c5bb2c60fd0d..4dc9ce1f33a7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -134,7 +134,6 @@ import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.user.UserAccount; -import com.cloud.user.dao.UserAccountDao; import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; import com.cloud.utils.Pair; @@ -147,7 +146,6 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.snapshot.VMSnapshotService; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -179,9 +177,6 @@ public class ServerAdapter extends ManagerBase { @Inject AccountService accountService; - @Inject - UserAccountDao userAccountDao; - @Inject DataCenterDao dataCenterDao; @@ -251,9 +246,6 @@ public class ServerAdapter extends ManagerBase { @Inject VMSnapshotDao vmSnapshotDao; - @Inject - VMSnapshotService vmSnapshotService; - //ToDo: check access on objects protected Role createServiceAccountRole() { @@ -960,21 +952,20 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { Pair serviceUserAccount = createServiceAccountIfNeeded(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { + DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for snapshot deletion"); + } + action = AsyncJobJoinVOToJobConverter.toAction(jobVo); if (async) { - DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); - ComponentContext.inject(cmd); - Map params = new HashMap<>(); - params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); - AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); - if (jobVo == null) { - throw new CloudRuntimeException("Failed to find job for snapshot deletion"); - } - action = AsyncJobJoinVOToJobConverter.toAction(jobVo); - } else { - vmSnapshotService.deleteVMSnapshot(vo.getId()); + // ToDo: wait for job completion? } } catch (Exception e) { throw new CloudRuntimeException("Failed to delete snapshot: " + e.getMessage(), e); From d9a7d2f097c0fa872ae1cd2148bdc4efb5fb5584 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Feb 2026 17:04:36 +0530 Subject: [PATCH 034/129] refactor, implement remaining endpoints Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 254 +++++++++-- .../cloudstack/veeam/api/ApiService.java | 50 ++- .../veeam/api/DataCentersRouteHandler.java | 3 +- .../veeam/api/DisksRouteHandler.java | 37 ++ .../cloudstack/veeam/api/VmsRouteHandler.java | 143 ++++++- .../AsyncJobJoinVOToJobConverter.java | 5 + .../converter/BackupVOToBackupConverter.java | 64 +++ .../ClusterVOToClusterConverter.java | 140 +++--- ...DataCenterJoinVOToDataCenterConverter.java | 32 +- .../converter/HostJoinVOToHostConverter.java | 7 +- ...ageTransferVOToImageTransferConverter.java | 5 +- .../api/converter/NicVOToNicConverter.java | 10 +- .../StoreVOToStorageDomainConverter.java | 140 +++--- .../converter/UserVmJoinVOToVmConverter.java | 84 ++-- .../VmSnapshotVOToSnapshotConverter.java | 4 +- .../VolumeJoinVOToDiskConverter.java | 94 ++-- .../cloudstack/veeam/api/dto/Actions.java | 10 +- .../apache/cloudstack/veeam/api/dto/Api.java | 86 +++- .../cloudstack/veeam/api/dto/ApiSummary.java | 42 +- .../cloudstack/veeam/api/dto/Backup.java | 65 ++- .../{SpecialObjectRef.java => Backups.java} | 20 +- .../cloudstack/veeam/api/dto/BaseDto.java | 2 +- .../apache/cloudstack/veeam/api/dto/Bios.java | 18 +- .../cloudstack/veeam/api/dto/BootMenu.java | 10 +- .../cloudstack/veeam/api/dto/Certificate.java | 4 - .../cloudstack/veeam/api/dto/Checkpoint.java | 76 ++++ .../dto/{OsVersion.java => Checkpoints.java} | 42 +- .../cloudstack/veeam/api/dto/Cluster.java | 402 ++++++++++++------ .../cloudstack/veeam/api/dto/Clusters.java | 12 +- .../apache/cloudstack/veeam/api/dto/Cpu.java | 22 +- .../cloudstack/veeam/api/dto/DataCenter.java | 111 ++++- .../cloudstack/veeam/api/dto/DataCenters.java | 14 +- .../apache/cloudstack/veeam/api/dto/Disk.java | 246 ++++++++--- .../veeam/api/dto/DiskAttachment.java | 97 ++++- .../veeam/api/dto/DiskAttachments.java | 16 +- .../cloudstack/veeam/api/dto/Disks.java | 12 +- .../cloudstack/veeam/api/dto/Fault.java | 16 +- .../veeam/api/dto/HardwareInformation.java | 10 - .../apache/cloudstack/veeam/api/dto/Host.java | 68 +-- .../veeam/api/dto/ImageTransfer.java | 37 +- .../apache/cloudstack/veeam/api/dto/Job.java | 10 +- .../apache/cloudstack/veeam/api/dto/Link.java | 24 +- .../cloudstack/veeam/api/dto/NamedList.java | 57 +++ .../cloudstack/veeam/api/dto/Network.java | 11 +- .../apache/cloudstack/veeam/api/dto/Nic.java | 26 +- .../apache/cloudstack/veeam/api/dto/Os.java | 8 +- .../cloudstack/veeam/api/dto/ProductInfo.java | 32 +- .../apache/cloudstack/veeam/api/dto/Ref.java | 18 +- .../veeam/api/dto/ReportedDevice.java | 26 +- .../cloudstack/veeam/api/dto/Snapshot.java | 6 +- .../veeam/api/dto/SpecialObjects.java | 22 +- .../cloudstack/veeam/api/dto/Storage.java | 54 ++- .../veeam/api/dto/StorageDomain.java | 246 ++++++++--- .../veeam/api/dto/StorageDomains.java | 9 +- .../veeam/api/dto/SummaryCount.java | 18 +- .../veeam/api/dto/SupportedVersions.java | 9 +- .../cloudstack/veeam/api/dto/Topology.java | 27 +- .../cloudstack/veeam/api/dto/Version.java | 54 ++- .../apache/cloudstack/veeam/api/dto/Vm.java | 214 ++++++++-- .../cloudstack/veeam/api/dto/VnicProfile.java | 20 +- 60 files changed, 2355 insertions(+), 1046 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/{SpecialObjectRef.java => Backups.java} (67%) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoint.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/{OsVersion.java => Checkpoints.java} (54%) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 4dc9ce1f33a7..d8efa2edbd75 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -36,7 +36,9 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; +import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; @@ -56,16 +58,19 @@ import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.ImageTransfer.Direction; import org.apache.cloudstack.backup.ImageTransfer.Format; import org.apache.cloudstack.backup.ImageTransferVO; import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.veeam.api.converter.AsyncJobJoinVOToJobConverter; +import org.apache.cloudstack.veeam.api.converter.BackupVOToBackupConverter; import org.apache.cloudstack.veeam.api.converter.ClusterVOToClusterConverter; import org.apache.cloudstack.veeam.api.converter.DataCenterJoinVOToDataCenterConverter; import org.apache.cloudstack.veeam.api.converter.HostJoinVOToHostConverter; @@ -77,6 +82,8 @@ import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; import org.apache.cloudstack.veeam.api.converter.VmSnapshotVOToSnapshotConverter; import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; +import org.apache.cloudstack.veeam.api.dto.Backup; +import org.apache.cloudstack.veeam.api.dto.Checkpoint; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.DataCenter; import org.apache.cloudstack.veeam.api.dto.Disk; @@ -86,7 +93,6 @@ import org.apache.cloudstack.veeam.api.dto.Job; import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.Nic; -import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.StorageDomain; @@ -246,6 +252,9 @@ public class ServerAdapter extends ManagerBase { @Inject VMSnapshotDao vmSnapshotDao; + @Inject + BackupDao backupDao; + //ToDo: check access on objects protected Role createServiceAccountRole() { @@ -353,7 +362,7 @@ public Cluster getCluster(String uuid) { } public List listAllHosts() { - final List hosts = hostJoinDao.listAll(); + final List hosts = hostJoinDao.listRoutingHostsByHypervisor(Hypervisor.HypervisorType.KVM); return HostJoinVOToHostConverter.toHostList(hosts); } @@ -410,11 +419,11 @@ public Vm createInstance(Vm request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); } - String name = request.name; + String name = request.getName(); Long zoneId = null; Long clusterId = null; - if (request.cluster != null && StringUtils.isNotEmpty(request.cluster.id)) { - ClusterVO clusterVO = clusterDao.findByUuid(request.cluster.id); + if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { + ClusterVO clusterVO = clusterDao.findByUuid(request.getCluster().getId()); if (clusterVO != null) { zoneId = clusterVO.getDataCenterId(); clusterId = clusterVO.getId(); @@ -425,14 +434,14 @@ public Vm createInstance(Vm request) { } Integer cpu = null; try { - cpu = request.cpu.topology.sockets; + cpu = request.getCpu().getTopology().getSockets(); } catch (Exception ignored) {} if (cpu == null) { throw new InvalidParameterValueException("CPU topology sockets must be specified"); } Long memory = null; try { - memory = Long.valueOf(request.memory); + memory = Long.valueOf(request.getMemory()); } catch (Exception ignored) {} if (memory == null) { throw new InvalidParameterValueException("Memory must be specified"); @@ -443,7 +452,7 @@ public Vm createInstance(Vm request) { } ApiConstants.BootType bootType = ApiConstants.BootType.BIOS; ApiConstants.BootMode bootMode = ApiConstants.BootMode.LEGACY; - if (request.bios != null && StringUtils.isNotEmpty(request.bios.type) && request.bios.type.contains("secure")) { + if (request.getBios() != null && StringUtils.isNotEmpty(request.getBios().getType()) && request.getBios().getType().contains("secure")) { bootType = ApiConstants.BootType.UEFI; bootMode = ApiConstants.BootMode.SECURE; } @@ -618,6 +627,40 @@ public Disk getDisk(String uuid) { return VolumeJoinVOToDiskConverter.toDisk(vo); } + public Disk copyDisk(String uuid) { + throw new InvalidParameterValueException("Copy Disk with ID " + uuid + " not implemented"); +// VolumeVO vo = volumeDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); +// } +// Pair serviceUserAccount = createServiceAccountIfNeeded(); +// CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); +// try { +// Volume volume = volumeApiService.copyVolume(vo.getId(), vo.getName() + "_copy", null, null); +// VolumeJoinVO copiedVolumeVO = volumeJoinDao.findById(volume.getId()); +// return VolumeJoinVOToDiskConverter.toDisk(copiedVolumeVO); +// } finally { +// CallContext.unregister(); +// } + } + + public Disk reduceDisk(String uuid) { + throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); +// VolumeVO vo = volumeDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); +// } +// Pair serviceUserAccount = createServiceAccountIfNeeded(); +// CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); +// try { +// Volume volume = volumeApiService.reduceDisk(vo.getId(), vo.getName() + "_copy", null, null); +// VolumeJoinVO copiedVolumeVO = volumeJoinDao.findById(volume.getId()); +// return VolumeJoinVOToDiskConverter.toDisk(copiedVolumeVO); +// } finally { +// CallContext.unregister(); +// } + } + protected List listDiskAttachmentsByInstanceId(final long instanceId) { List kvmVolumes = volumeJoinDao.listByInstanceId(instanceId); return VolumeJoinVOToDiskConverter.toDiskAttachmentList(kvmVolumes); @@ -636,12 +679,12 @@ public DiskAttachment handleInstanceAttachDisk(final String vmUuid, final DiskAt if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - if (request == null || request.disk == null || StringUtils.isEmpty(request.disk.id)) { + if (request == null || request.getDisk() == null || StringUtils.isEmpty(request.getDisk().getId())) { throw new InvalidParameterValueException("Request disk data is empty"); } - VolumeVO volumeVO = volumeDao.findByUuid(request.disk.id); + VolumeVO volumeVO = volumeDao.findByUuid(request.getDisk().getId()); if (volumeVO == null) { - throw new InvalidParameterValueException("Disk with ID " + request.disk.id + " not found"); + throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } Pair serviceUserAccount = createServiceAccountIfNeeded(); CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); @@ -666,23 +709,23 @@ public Disk handleCreateDisk(Disk request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); } - String name = request.name; + String name = request.getName(); if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { throw new InvalidParameterValueException("Only worker VM disk creation is supported"); } - if (request.storageDomains == null || CollectionUtils.isEmpty(request.storageDomains.storageDomain) || - request.storageDomains.storageDomain.size() > 1) { + if (request.getStorageDomains() == null || CollectionUtils.isEmpty(request.getStorageDomains().getStorageDomain()) || + request.getStorageDomains().getStorageDomain().size() > 1) { throw new InvalidParameterValueException("Exactly one storage domain must be specified"); } - Ref domain = request.storageDomains.storageDomain.get(0); - if (domain == null || domain.id == null) { + StorageDomain domain = request.getStorageDomains().getStorageDomain().get(0); + if (domain == null || domain.getId() == null) { throw new InvalidParameterValueException("Storage domain ID must be specified"); } - StoragePoolVO pool = primaryDataStoreDao.findByUuid(domain.id); + StoragePoolVO pool = primaryDataStoreDao.findByUuid(domain.getId()); if (pool == null) { - throw new InvalidParameterValueException("Storage domain with ID " + domain.id + " not found"); + throw new InvalidParameterValueException("Storage domain with ID " + domain.getId() + " not found"); } - String sizeStr = request.provisionedSize; + String sizeStr = request.getProvisionedSize(); if (StringUtils.isBlank(sizeStr)) { throw new InvalidParameterValueException("Provisioned size must be specified"); } @@ -697,9 +740,9 @@ public Disk handleCreateDisk(Disk request) { } provisionedSizeInGb = Math.max(1L, provisionedSizeInGb / (1024L * 1024L * 1024L)); Long initialSize = null; - if (StringUtils.isNotBlank(request.initialSize)) { + if (StringUtils.isNotBlank(request.getInitialSize())) { try { - initialSize = Long.parseLong(request.initialSize); + initialSize = Long.parseLong(request.getInitialSize()); } catch (NumberFormatException ignored) {} } Pair serviceUserAccount = createServiceAccountIfNeeded(); @@ -763,12 +806,12 @@ public Nic handleAttachInstanceNic(final String vmUuid, final Nic request) { if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - if (request == null || request.getVnicProfile() == null || StringUtils.isEmpty(request.getVnicProfile().id)) { + if (request == null || request.getVnicProfile() == null || StringUtils.isEmpty(request.getVnicProfile().getId())) { throw new InvalidParameterValueException("Request nic data is empty"); } - NetworkVO networkVO = networkDao.findByUuid(request.getVnicProfile().id); + NetworkVO networkVO = networkDao.findByUuid(request.getVnicProfile().getId()); if (networkVO == null) { - throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().id+ " not found"); + throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().getId() + " not found"); } Pair serviceUserAccount = createServiceAccountIfNeeded(); CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); @@ -808,12 +851,12 @@ public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { if (request == null) { throw new InvalidParameterValueException("Request image transfer data is empty"); } - if (request.getDisk() == null || StringUtils.isBlank(request.getDisk().id)) { + if (request.getDisk() == null || StringUtils.isBlank(request.getDisk().getId())) { throw new InvalidParameterValueException("Disk ID must be specified"); } - VolumeJoinVO volumeVO = volumeJoinDao.findByUuid(request.getDisk().id); + VolumeJoinVO volumeVO = volumeJoinDao.findByUuid(request.getDisk().getId()); if (volumeVO == null) { - throw new InvalidParameterValueException("Disk with ID " + request.getDisk().id + " not found"); + throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); if (direction == null) { @@ -974,4 +1017,161 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { } return action; } + + public ResourceAction revertToSnapshot(String uuid) { + throw new InvalidParameterValueException("revertToSnapshot with ID " + uuid + " not implemented"); +// ResourceAction action = null; +// VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); +// } +// Pair serviceUserAccount = createServiceAccountIfNeeded(); +// CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); +// try { +// RevertToVMSnapshotCmd cmd = new RevertToVMSnapshotCmd(); +// ComponentContext.inject(cmd); +// Map params = new HashMap<>(); +// params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); +// ApiServerService.AsyncCmdResult result = +// apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), +// serviceUserAccount.second()); +// AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); +// if (jobVo == null) { +// throw new CloudRuntimeException("Failed to find job for snapshot revert"); +// } +// action = AsyncJobJoinVOToJobConverter.toAction(jobVo); +// } catch (Exception e) { +// throw new CloudRuntimeException("Failed to revert to snapshot: " + e.getMessage(), e); +// } finally { +// CallContext.unregister(); +// } +// return action; + } + + public List listBackupsByInstanceUuid(final String uuid) { + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + List backups = backupDao.searchByVmIds(List.of(vo.getId())); + return BackupVOToBackupConverter.toBackupList(backups, id -> vo); + } + + public Backup createInstanceBackup(final String vmUuid, final Backup request) { + UserVmVO vmVo = userVmDao.findByUuid(vmUuid); + if (vmVo == null) { + throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); + } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + CreateBackupCmd cmd = new CreateBackupCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); + params.put(ApiConstants.NAME, request.getName()); + params.put(ApiConstants.DESCRIPTION, request.getDescription()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + if (result.objectId == null) { + throw new CloudRuntimeException("No backup ID returned"); + } + BackupVO vo = backupDao.findById(result.objectId); + if (vo == null) { + throw new CloudRuntimeException("Backup not found"); + } + return BackupVOToBackupConverter.toBackup(vo, id -> vmVo); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to create backup: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } + } + + public Backup getBackup(String uuid) { + BackupVO vo = backupDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); + } + return BackupVOToBackupConverter.toBackup(vo, id -> userVmDao.findById(id)); + } + + public List listDisksByBackupUuid(final String uuid) { + throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implmenented"); +// BackupVO vo = backupDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); +// } +// return VolumeJoinVOToDiskConverter.toDiskList(volumes); + } + + public void finalizeBackup(final String vmUuid, final String uuid, String data) { + ResourceAction action = null; + UserVmVO vmVo = userVmDao.findByUuid(vmUuid); + if (vmVo == null) { + throw new InvalidParameterValueException("Instance with ID " + vmUuid + " not found"); + } + BackupVO vo = backupDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); + } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + FinalizeBackupCmd cmd = new FinalizeBackupCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); + params.put(ApiConstants.BACKUP_ID, vo.getUuid()); + boolean result = incrementalBackupService.finalizeBackup(cmd); + if (!result) { + throw new CloudRuntimeException("Failed to finalize backup"); + } + } catch (Exception e) { + throw new CloudRuntimeException("Failed to finalize backup: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } + } + + public List listCheckpointsByInstanceUuid(final String uuid) { + throw new InvalidParameterValueException("Checkpoints for VM with ID " + uuid + " not implemented"); +// UserVmVO vo = userVmDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); +// } +// List checkpoints = checkpointDao.findByVmId(vo.getId()); +// return CheckpointVOToCheckpointConverter.toCheckpointList(checkpoints, vo.getUuid()); + } + + public ResourceAction deleteCheckpoint(String uuid, boolean async) { + throw new InvalidParameterValueException("Delete Checkpoint with ID " + uuid + " not implemented"); +// ResourceAction action = null; +// CheckpointVO vo = checkpointDao.findByUuid(uuid); +// if (vo == null) { +// throw new InvalidParameterValueException("Checkpoint with ID " + uuid + " not found"); +// } +// Pair serviceUserAccount = createServiceAccountIfNeeded(); +// CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); +// try { +// DeleteCheckpointCmd cmd = new DeleteCheckpointCmd(); +// ComponentContext.inject(cmd); +// Map params = new HashMap<>(); +// params.put(ApiConstants.CHECKPOINT_ID, vo.getUuid()); +// ApiServerService.AsyncCmdResult result = +// apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), +// serviceUserAccount.second()); +// AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); +// if (jobVo == null) { +// throw new CloudRuntimeException("Failed to find job for checkpoint deletion"); +// } +// action = AsyncJobJoinVOToJobConverter.toAction(jobVo); +// } catch (Exception e) { +// throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); +// } finally { +// CallContext.unregister(); +// } +// return action; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index dd0e4b250826..fbe666882df2 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -19,8 +19,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -37,7 +35,6 @@ import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.ProductInfo; import org.apache.cloudstack.veeam.api.dto.Ref; -import org.apache.cloudstack.veeam.api.dto.SpecialObjectRef; import org.apache.cloudstack.veeam.api.dto.SpecialObjects; import org.apache.cloudstack.veeam.api.dto.SummaryCount; import org.apache.cloudstack.veeam.api.dto.Version; @@ -94,58 +91,59 @@ private static Api createDummyApi(String basePath) { add(links, basePath + "/disks", "disks"); add(links, basePath + "/disks?search={query}", "disks/search"); - api.link = links; + api.setLink(links); /* ---------------- Engine backup ---------------- */ - api.engineBackup = new EmptyElement(); + api.setEngineBackup(new EmptyElement()); /* ---------------- Product info ---------------- */ ProductInfo productInfo = new ProductInfo(); - productInfo.instanceId = UuidUtils.nameUUIDFromBytes(VeeamControlService.BindAddress.value().getBytes(StandardCharsets.UTF_8)).toString(); + productInfo.setInstanceId(UuidUtils.nameUUIDFromBytes( + VeeamControlService.BindAddress.value().getBytes(StandardCharsets.UTF_8)).toString()); productInfo.name = "oVirt Engine"; Version version = new Version(); - version.build = "8"; - version.fullVersion = "4.5.8-0.master.fake.el9"; - version.major = 4; - version.minor = 5; - version.revision = 0; + version.setBuild("8"); + version.setFullVersion("4.5.8-0.master.fake.el9"); + version.setMajor(4); + version.setMinor(5); + version.setRevision(0); productInfo.version = version; - api.productInfo = productInfo; + api.setProductInfo(productInfo); /* ---------------- Special objects ---------------- */ SpecialObjects specialObjects = new SpecialObjects(); - specialObjects.blankTemplate = new SpecialObjectRef( + specialObjects.setBlankTemplate(Ref.of( basePath + "/templates/00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000000" - ); - specialObjects.rootTag = new SpecialObjectRef( + )); + specialObjects.setRootTag(Ref.of( basePath + "/tags/00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000000" - ); - api.specialObjects = specialObjects; + )); + api.setSpecialObjects(specialObjects); /* ---------------- Summary ---------------- */ ApiSummary summary = new ApiSummary(); - summary.hosts = new SummaryCount(1, 1); - summary.storageDomains = new SummaryCount(1, 2); - summary.users = new SummaryCount(1, 1); - summary.vms = new SummaryCount(1, 8); - api.summary = summary; + summary.setHosts(new SummaryCount(1, 1)); + summary.setStorageDomains(new SummaryCount(1, 2)); + summary.setUsers(new SummaryCount(1, 1)); + summary.setVms(new SummaryCount(1, 8)); + api.setSummary(summary); /* ---------------- Time ---------------- */ - api.time = OffsetDateTime.now(ZoneOffset.ofHours(2)).toInstant().toEpochMilli(); + api.setTime(System.currentTimeMillis()); /* ---------------- Users ---------------- */ String userId = UUID.randomUUID().toString(); - api.authenticatedUser = Ref.of(basePath + "/users/" + userId, userId); - api.effectiveUser = Ref.of(basePath + "/users/" + userId, userId); + api.setAuthenticatedUser(Ref.of(basePath + "/users/" + userId, userId)); + api.setEffectiveUser(Ref.of(basePath + "/users/" + userId, userId)); return api; } private static void add(List links, String href, String rel) { - links.add(new Link(href, rel)); + links.add(Link.of(href, rel)); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index dd324eb9ee3e..1b9e2e014014 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -118,7 +118,8 @@ protected void handleGetStorageDomainsByDcId(final String id, final HttpServletR final VeeamControlServlet io) throws IOException { try { List storageDomains = serverAdapter.listStorageDomainsByDcId(id); - StorageDomains response = new StorageDomains(storageDomains); + StorageDomains response = new StorageDomains(); + response.setStorageDomain(storageDomains); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index fa1248539b1b..c13bacdfba0a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -90,6 +90,23 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleDeleteById(id, resp, outFormat, io); return; } + } else if (idAndSubPath.size() == 2) { + String subPath = idAndSubPath.get(1); + if ("copy".equals(subPath)) { + if ("POST".equalsIgnoreCase(method)) { + handlePostDiskCopy(id, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; + } else if ("reduce".equals(subPath)) { + if ("POST".equalsIgnoreCase(method)) { + handlePostDiskReduce(id, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; + } } } @@ -136,4 +153,24 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, io.badRequest(resp, e.getMessage(), outFormat); } } + + protected void handlePostDiskCopy(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + Disk response = serverAdapter.copyDisk(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handlePostDiskReduce(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + Disk response = serverAdapter.reduceDisk(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 908aece8bdfb..9eb12fdf3968 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -28,8 +28,14 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; +import org.apache.cloudstack.veeam.api.dto.Backup; +import org.apache.cloudstack.veeam.api.dto.Checkpoint; +import org.apache.cloudstack.veeam.api.dto.Checkpoints; +import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.DiskAttachments; +import org.apache.cloudstack.veeam.api.dto.Disks; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.ResourceAction; @@ -164,6 +170,22 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handlePostSnapshotForVmId(id, req, resp, outFormat, io); } return; + } else if ("backups".equals(subPath)) { + if (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, POST", outFormat); + } else if ("GET".equalsIgnoreCase(method)) { + handleGetBackupsByVmId(id, resp, outFormat, io); + } else if ("POST".equalsIgnoreCase(method)) { + handlePostBackupForVmId(id, req, resp, outFormat, io); + } + return; + } else if ("checkpoints".equals(subPath)) { + if ("GET".equalsIgnoreCase(method)) { + handleGetCheckpointsByVmId(id, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "GET, POST", outFormat); + } + return; } } else if (idAndSubPath.size() == 3) { String subPath = idAndSubPath.get(1); @@ -172,11 +194,25 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path if (!"GET".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { io.methodNotAllowed(resp, "GET, DELETE", outFormat); } else if ("GET".equalsIgnoreCase(method)) { - handleGetSnapshotsById(subId, resp, outFormat, io); + handleGetSnapshotById(subId, resp, outFormat, io); } else if ("DELETE".equalsIgnoreCase(method)) { handleDeleteSnapshotById(subId, req, resp, outFormat, io); } return; + } else if ("backups".equals(subPath)) { + if ("GET".equalsIgnoreCase(method)) { + handleGetBackupById(subId, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "GET", outFormat); + } + return; + } else if ("checkpoints".equals(subPath)) { + if ("DELETE".equalsIgnoreCase(method)) { + handleDeleteCheckpointById(subId, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "DELETE", outFormat); + } + return; } } else if (idAndSubPath.size() == 4) { String subPath = idAndSubPath.get(1); @@ -189,6 +225,20 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.methodNotAllowed(resp, "POST", outFormat); } return; + } else if ("backups".equals(subPath) && "disks".equals(action)) { + if ("GET".equalsIgnoreCase(method)) { + handleGetBackupDisksById(subId, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "GET", outFormat); + } + return; + } else if ("backups".equals(subPath) && "finalize".equals(action)) { + if ("POST".equalsIgnoreCase(method)) { + handleFinalizeBackupById(id, subId, req, resp, outFormat, io); + } else { + io.methodNotAllowed(resp, "POST", outFormat); + } + return; } } } @@ -405,8 +455,8 @@ protected void handlePostSnapshotForVmId(final String id, final HttpServletReque } } - protected void handleGetSnapshotsById(final String id, final HttpServletResponse resp, - final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + protected void handleGetSnapshotById(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { Snapshot response = serverAdapter.getSnapshot(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); @@ -435,7 +485,94 @@ protected void handleDeleteSnapshotById(final String id, final HttpServletReques protected void handleRestoreSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + //ToDo: implement String data = getRequestData(req); io.badRequest(resp, "Not implemented", outFormat); } + + protected void handleGetBackupsByVmId(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + List backups = serverAdapter.listBackupsByInstanceUuid(id); + NamedList response = NamedList.of("backups", backups); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } + + protected void handlePostBackupForVmId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = getRequestData(req); + try { + Backup request = io.getMapper().jsonMapper().readValue(data, Backup.class); + Backup response = serverAdapter.createInstanceBackup(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handleGetBackupById(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + Backup response = serverAdapter.getBackup(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } + + protected void handleGetBackupDisksById(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + try { + List disks = serverAdapter.listDisksByBackupUuid(id); + Disks response = new Disks(disks); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } + + protected void handleFinalizeBackupById(final String vmId, final String backupId, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String data = getRequestData(req); + try { + serverAdapter.finalizeBackup(vmId, backupId, data); + io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + + protected void handleGetCheckpointsByVmId(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + try { + List checkpoints = serverAdapter.listCheckpointsByInstanceUuid(id); + Checkpoints response = new Checkpoints(checkpoints); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } + + protected void handleDeleteCheckpointById(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { + String asyncStr = req.getParameter("async"); + boolean async = !Boolean.FALSE.toString().equals(asyncStr); + try { + ResourceAction action = serverAdapter.deleteCheckpoint(id, async); + if (action != null) { + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, action, outFormat); + } else { + io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); + } + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index 6c273a22f286..c66e9f78d0f7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -64,6 +64,7 @@ public static Job toJob(AsyncJobJoinVO vo) { job.setLastUpdated(System.currentTimeMillis()); job.setStartTime(vo.getCreated().getTime()); JobInfo.Status status = JobInfo.Status.values()[vo.getStatus()]; + Long endTime = System.currentTimeMillis(); if (status == JobInfo.Status.SUCCEEDED) { job.setStatus("finished"); job.setEndTime(System.currentTimeMillis()); @@ -73,6 +74,10 @@ public static Job toJob(AsyncJobJoinVO vo) { job.setStatus("aborted"); } else { job.setStatus("started"); + endTime = null; + } + if (endTime != null) { + job.setEndTime(endTime); } job.setOwner(Ref.of(basePath + "/api/users/" + vo.getUserUuid(), vo.getUserUuid())); job.setActions(new Actions()); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java new file mode 100644 index 000000000000..5d93524ef528 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cloudstack.backup.BackupVO; +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.Backup; +import org.apache.cloudstack.veeam.api.dto.Vm; + +import com.cloud.vm.UserVmVO; + +public class BackupVOToBackupConverter { + + public static Backup toBackup(final BackupVO backupVO, final Function vmResolver) { + Backup backup = new Backup(); + final String basePath = VeeamControlService.ContextPath.value(); + backup.setHref(basePath + VmsRouteHandler.BASE_ROUTE + "/backups/" + backupVO.getUuid()); + backup.setId(backupVO.getUuid()); + backup.setName(backupVO.getName()); + backup.setDescription(backupVO.getDescription()); + backup.setCreationDate(backupVO.getDate().getTime()); +// backup.setPhase(backupVO.getPhase().name()); +// if (backupVO.getFromCheckpointId() != null) { +// backup.setFromCheckpointId(backupVO.getFromCheckpointId().toString()); +// } +// if (backupVO.getToCheckpointId() != null) { +// backup.setToCheckpointId(backupVO.getToCheckpointId().toString()); +// } + if (vmResolver != null) { + final UserVmVO vmVO = vmResolver.apply(backupVO.getVmId()); + if (vmVO != null) { + backup.setVm(Vm.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmVO.getUuid(), vmVO.getUuid())); + } + } + return backup; + } + + public static List toBackupList(final List backupVOs, final Function vmResolver) { + return backupVOs + .stream() + .map(backupVO -> toBackup(backupVO, vmResolver)) + .collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java index 3a2c9be5b486..44789f694bdb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -27,8 +27,10 @@ import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Cluster; +import org.apache.cloudstack.veeam.api.dto.Cpu; import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Version; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.dc.ClusterVO; @@ -43,115 +45,113 @@ public static Cluster toCluster(final ClusterVO vo, final Function" final String clusterId = vo.getUuid(); - c.id = clusterId; - c.href = basePath + ClustersRouteHandler.BASE_ROUTE + "/" + clusterId; + c.setId(clusterId); + c.setHref(basePath + ClustersRouteHandler.BASE_ROUTE + "/" + clusterId); - c.name = vo.getName(); - c.description = vo.getName(); - c.comment = ""; + c.setName(vo.getName()); // --- sensible defaults (match your sample) - c.ballooningEnabled = "true"; - c.biosType = "q35_ovmf"; // or "q35_secure_boot" if you want to align with VM BIOS you saw - c.fipsMode = "disabled"; - c.firewallType = "firewalld"; - c.glusterService = "false"; - c.haReservation = "false"; - c.switchType = "legacy"; - c.threadsAsCores = "false"; - c.trustedService = "false"; - c.tunnelMigration = "false"; - c.upgradeInProgress = "false"; - c.upgradePercentComplete = "0"; - c.virtService = "true"; - c.vncEncryption = "false"; - c.logMaxMemoryUsedThreshold = "95"; - c.logMaxMemoryUsedThresholdType = "percentage"; + c.setBallooningEnabled("true"); + c.setBiosType("q35_ovmf"); // or "q35_secure_boot" if you want to align with VM BIOS you saw + c.setFipsMode("disabled"); + c.setFirewallType("firewalld"); + c.setGlusterService("false"); + c.setHaReservation("false"); + c.setSwitchType("legacy"); + c.setThreadsAsCores("false"); + c.setTrustedService("false"); + c.setTunnelMigration("false"); + c.setUpgradeInProgress("false"); + c.setUpgradePercentComplete("0"); + c.setVirtService("true"); + c.setVncEncryption("false"); + c.setLogMaxMemoryUsedThreshold("95"); + c.setLogMaxMemoryUsedThresholdType("percentage"); // --- cpu (best-effort defaults) - final Cluster.ClusterCpu cpu = new Cluster.ClusterCpu(); - cpu.architecture = "x86_64"; - cpu.type = "x86_64"; // replace if you can detect host cpu model - c.cpu = cpu; + final Cpu cpu = new Cpu(); + cpu.setArchitecture("x86_64"); + cpu.setType("x86_64"); // replace if you can detect host cpu model + c.setCpu(cpu); // --- version (ovirt engine version; keep fixed unless you want to expose something else) - final Cluster.Version ver = new Cluster.Version(); - ver.major = "4"; - ver.minor = "8"; - c.version = ver; + final Version ver = new Version(); + ver.setMajor(4); + ver.setMinor(8); + c.setVersion(ver); // --- ksm / memory policy (defaults) - c.ksm = new Cluster.Ksm(); - c.ksm.enabled = "true"; - c.ksm.mergeAcrossNodes = "true"; + c.setKsm(new Cluster.Ksm()); + c.getKsm().enabled = "true"; + c.getKsm().mergeAcrossNodes = "true"; - c.memoryPolicy = new Cluster.MemoryPolicy(); - c.memoryPolicy.overCommit = new Cluster.OverCommit(); - c.memoryPolicy.overCommit.percent = "100"; - c.memoryPolicy.transparentHugepages = new Cluster.TransparentHugepages(); - c.memoryPolicy.transparentHugepages.enabled = "true"; + c.setMemoryPolicy(new Cluster.MemoryPolicy()); + c.getMemoryPolicy().overCommit = new Cluster.OverCommit(); + c.getMemoryPolicy().overCommit.percent = "100"; + c.getMemoryPolicy().transparentHugepages = new Cluster.TransparentHugepages(); + c.getMemoryPolicy().transparentHugepages.enabled = "true"; // --- migration defaults - c.migration = new Cluster.Migration(); - c.migration.autoConverge = "inherit"; - c.migration.bandwidth = new Cluster.Bandwidth(); - c.migration.bandwidth.assignmentMethod = "auto"; - c.migration.compressed = "inherit"; - c.migration.encrypted = "inherit"; - c.migration.parallelMigrationsPolicy = "disabled"; + c.setMigration(new Cluster.Migration()); + c.getMigration().autoConverge = "inherit"; + c.getMigration().bandwidth = new Cluster.Bandwidth(); + c.getMigration().bandwidth.assignmentMethod = "auto"; + c.getMigration().compressed = "inherit"; + c.getMigration().encrypted = "inherit"; + c.getMigration().parallelMigrationsPolicy = "disabled"; // policy ref (dummy but valid shape) - c.migration.policy = Ref.of(basePath + "/migrationpolicies/" + stableUuid("migrationpolicy:default"), + c.getMigration().policy = Ref.of(basePath + "/migrationpolicies/" + stableUuid("migrationpolicy:default"), stableUuid("migrationpolicy:default") ); // --- rng sources - c.requiredRngSources = new Cluster.RequiredRngSources(); - c.requiredRngSources.requiredRngSource = Collections.singletonList("urandom"); + c.setRequiredRngSources(new Cluster.RequiredRngSources()); + c.getRequiredRngSources().requiredRngSource = Collections.singletonList("urandom"); // --- error handling - c.errorHandling = new Cluster.ErrorHandling(); - c.errorHandling.onError = "migrate"; + c.setErrorHandling(new Cluster.ErrorHandling()); + c.getErrorHandling().onError = "migrate"; // --- fencing policy defaults - c.fencingPolicy = new Cluster.FencingPolicy(); - c.fencingPolicy.enabled = "true"; - c.fencingPolicy.skipIfConnectivityBroken = new Cluster.SkipIfConnectivityBroken(); - c.fencingPolicy.skipIfConnectivityBroken.enabled = "false"; - c.fencingPolicy.skipIfConnectivityBroken.threshold = "50"; - c.fencingPolicy.skipIfGlusterBricksUp = "false"; - c.fencingPolicy.skipIfGlusterQuorumNotMet = "false"; - c.fencingPolicy.skipIfSdActive = new Cluster.SkipIfSdActive(); - c.fencingPolicy.skipIfSdActive.enabled = "false"; + c.setFencingPolicy(new Cluster.FencingPolicy()); + c.getFencingPolicy().enabled = "true"; + c.getFencingPolicy().skipIfConnectivityBroken = new Cluster.SkipIfConnectivityBroken(); + c.getFencingPolicy().skipIfConnectivityBroken.enabled = "false"; + c.getFencingPolicy().skipIfConnectivityBroken.threshold = "50"; + c.getFencingPolicy().skipIfGlusterBricksUp = "false"; + c.getFencingPolicy().skipIfGlusterQuorumNotMet = "false"; + c.getFencingPolicy().skipIfSdActive = new Cluster.SkipIfSdActive(); + c.getFencingPolicy().skipIfSdActive.enabled = "false"; // --- scheduling policy props (optional; dummy ok) - c.customSchedulingPolicyProperties = new Cluster.CustomSchedulingPolicyProperties(); + c.setCustomSchedulingPolicyProperties(new Cluster.CustomSchedulingPolicyProperties()); final Cluster.Property p1 = new Cluster.Property(); p1.name = "HighUtilization"; p1.value = "80"; final Cluster.Property p2 = new Cluster.Property(); p2.name = "CpuOverCommitDurationMinutes"; p2.value = "2"; - c.customSchedulingPolicyProperties.property = List.of(p1, p2); + c.getCustomSchedulingPolicyProperties().property = List.of(p1, p2); // --- data_center ref mapping (CloudStack cluster -> pod -> zone) if (dataCenterResolver != null) { final DataCenterJoinVO zone = dataCenterResolver.apply(vo.getDataCenterId()); if (zone != null) { - c.dataCenter = Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + zone.getUuid(), zone.getUuid()); + c.setDataCenter(Ref.of(basePath + DataCentersRouteHandler.BASE_ROUTE + "/" + zone.getUuid(), zone.getUuid())); } } // --- mac pool & scheduling policy refs (dummy but consistent) - c.macPool = Ref.of(basePath + "/macpools/" + stableUuid("macpool:default"), - stableUuid("macpool:default")); - c.schedulingPolicy = Ref.of(basePath + "/schedulingpolicies/" + stableUuid("schedpolicy:default"), - stableUuid("schedpolicy:default")); + c.setMacPool(Ref.of(basePath + "/macpools/" + stableUuid("macpool:default"), + stableUuid("macpool:default"))); + c.setSchedulingPolicy(Ref.of(basePath + "/schedulingpolicies/" + stableUuid("schedpolicy:default"), + stableUuid("schedpolicy:default"))); // --- actions.links (can be omitted; but Veeam sometimes expects actions to exist) final Actions actions = new Actions(); - actions.link = Collections.emptyList(); - c.actions = actions; + actions.setLink(Collections.emptyList()); + c.setActions(actions); // --- related links (optional) - c.link = List.of( - new Link("networks", c.href + "/networks") - ); + c.setLink(List.of( + Link.of("networks", c.getHref() + "/networks") + )); return c; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java index 465420fc9841..0cb160a7dd2c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java @@ -41,32 +41,32 @@ public static DataCenter toDataCenter(final DataCenterJoinVO zone) { final DataCenter dc = new DataCenter(); // ---- Identity ---- - dc.id = id; - dc.href = href; - dc.name = zone.getName(); - dc.description = zone.getDescription(); + dc.setId(id); + dc.setHref(href); + dc.setName(zone.getName()); + dc.setDescription(zone.getDescription()); // ---- State ---- - dc.status = Grouping.AllocationState.Enabled.equals(zone.getAllocationState()) ? "up" : "down"; - dc.local = "false"; - dc.quotaMode = "disabled"; - dc.storageFormat = "v5"; + dc.setStatus(Grouping.AllocationState.Enabled.equals(zone.getAllocationState()) ? "up" : "down"); + dc.setLocal("false"); + dc.setQuotaMode("disabled"); + dc.setStorageFormat("v5"); // ---- Versions (static but valid) ---- final Version v48 = new Version(); - v48.major = 4; - v48.minor = 8; - dc.version = v48; - dc.supportedVersions = new SupportedVersions(List.of(v48)); + v48.setMajor(4); + v48.setMinor(8); + dc.setVersion(v48); + dc.setSupportedVersions(new SupportedVersions(List.of(v48))); // ---- mac_pool (static placeholder) ---- - dc.macPool = Ref.of(basePath + "/macpools/default","default"); + dc.setMacPool(Ref.of(basePath + "/macpools/default", "default")); // ---- Related links ---- dc.link = Arrays.asList( - new Link(href + "/clusters", "clusters"), - new Link(href + "/networks", "networks"), - new Link(href + "/storagedomains", "storagedomains") + Link.of(href + "/clusters", "clusters"), + Link.of(href + "/networks", "networks"), + Link.of(href + "/storagedomains", "storagedomains") ); return dc; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java index 32c9c3040e91..d36e5ce73714 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java @@ -68,12 +68,7 @@ public static Host toHost(final HostJoinVO vo) { final Cpu cpu = new Cpu(); - final Topology topo = new Topology(); - // oVirt topology: sockets/cores/threads. We approximate. - // If CloudStack has cpuNumber = total cores, treat as sockets count w/ 1 core, 1 thread. - topo.sockets = vo.getCpuSockets(); - topo.cores = vo.getCpus(); - topo.threads = 1; + final Topology topo = new Topology(vo.getCpuSockets(), vo.getCpus(), 1); // --- Memory --- h.setMemory(String.valueOf(vo.getTotalMemory())); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java index 5fc4313bdb1c..fa4d608ee711 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java @@ -85,9 +85,6 @@ public static List toImageTransferList(List toStorageDomainListFromStores(final List type=glance // If you want "nfs" for secondary, map based on provider/protocol instead. - st.type = mapImageStorageType(store); + st.setType(mapImageStorageType(store)); - if ("nfs".equals(st.type)) { - st.address = ""; - st.path = ""; - st.mountOptions = ""; - st.nfsVersion = "auto"; + if ("nfs".equals(st.getType())) { + st.setAddress(""); + st.setPath(""); + st.setMountOptions(""); + st.setNfsVersion("auto"); } return st; } @@ -188,19 +184,19 @@ private static List defaultStorageDomainLinks(String basePath, boolean inc // Mirrors the rels you pasted; keep stable order. // You can add/remove based on what endpoints you actually implement. List common = new java.util.ArrayList<>(); - common.add(new Link("diskprofiles", href(basePath, "/diskprofiles"))); + common.add(Link.of("diskprofiles", href(basePath, "/diskprofiles"))); if (includeDisks) { - common.add(new Link("disks", href(basePath, "/disks"))); - common.add(new Link("storageconnections", href(basePath, "/storageconnections"))); + common.add(Link.of("disks", href(basePath, "/disks"))); + common.add(Link.of("storageconnections", href(basePath, "/storageconnections"))); } - common.add(new Link("permissions", href(basePath, "/permissions"))); + common.add(Link.of("permissions", href(basePath, "/permissions"))); if (includeTemplates) { - common.add(new Link("templates", href(basePath, "/templates"))); - common.add(new Link("vms", href(basePath, "/vms"))); + common.add(Link.of("templates", href(basePath, "/templates"))); + common.add(Link.of("vms", href(basePath, "/vms"))); } else { - common.add(new Link("images", href(basePath, "/images"))); + common.add(Link.of("images", href(basePath, "/images"))); } - common.add(new Link("disksnapshots", href(basePath, "/disksnapshots"))); + common.add(Link.of("disksnapshots", href(basePath, "/disksnapshots"))); return common; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 15d7071e9597..c119ac072273 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -65,18 +65,18 @@ public static Vm toVm(final UserVmJoinVO src, final Function h final String basePath = VeeamControlService.ContextPath.value(); final Vm dst = new Vm(); - dst.id = src.getUuid(); - dst.name = StringUtils.firstNonBlank(src.getName(), src.getInstanceName()); + dst.setId(src.getUuid()); + dst.setName(StringUtils.firstNonBlank(src.getName(), src.getInstanceName())); // CloudStack doesn't really have "description" for VM; displayName is closest - dst.description = src.getDisplayName(); - dst.href = basePath + VmsRouteHandler.BASE_ROUTE + "/" + src.getUuid(); - dst.status = mapStatus(src.getState()); + dst.setDescription(src.getDisplayName()); + dst.setHref(basePath + VmsRouteHandler.BASE_ROUTE + "/" + src.getUuid()); + dst.setStatus(mapStatus(src.getState())); dst.setCreationTime(src.getCreated().getTime()); final Date lastUpdated = src.getLastUpdated() != null ? src.getLastUpdated() : src.getCreated(); - if ("down".equals(dst.status)) { - dst.stopTime = lastUpdated.getTime(); + if ("down".equals(dst.getStatus())) { + dst.setStopTime(lastUpdated.getTime()); } - if ("up".equals(dst.status)) { + if ("up".equals(dst.getStatus())) { dst.setStartTime(lastUpdated.getTime()); } final Ref template = buildRef( @@ -84,40 +84,45 @@ public static Vm toVm(final UserVmJoinVO src, final Function h "templates", src.getTemplateUuid() ); - dst.template = template; - dst.originalTemplate = template; + dst.setTemplate(template); + dst.setOriginalTemplate(template); if (StringUtils.isNotBlank(src.getHostUuid())) { - dst.host = buildRef( + dst.setHost(buildRef( basePath + ApiService.BASE_ROUTE, "hosts", - src.getHostUuid()); + src.getHostUuid())); } if (hostResolver != null) { HostJoinVO hostVo = hostResolver.apply(src.getHostId() == null ? src.getLastHostId() : src.getHostId()); if (hostVo != null) { - dst.host = buildRef( + dst.setHost(buildRef( basePath + ApiService.BASE_ROUTE, "hosts", - hostVo.getUuid()); - dst.cluster = buildRef( + hostVo.getUuid())); + dst.setCluster(buildRef( basePath + ApiService.BASE_ROUTE, "clusters", - hostVo.getClusterUuid()); + hostVo.getClusterUuid())); } } - dst.memory = String.valueOf(src.getRamSize() * 1024L * 1024L); - - dst.cpu = new Cpu(src.getArch(), new Topology(src.getCpu(), 1, 1)); - dst.os = new Os(); - dst.os.type = src.getGuestOsId() % 2 == 0 + dst.setMemory(String.valueOf(src.getRamSize() * 1024L * 1024L)); + Cpu cpu = new Cpu(); + cpu.setArchitecture(src.getArch()); + cpu.setTopology(new Topology(src.getCpu(), 1, 1)); + dst.setCpu(cpu); + Os os = new Os(); + os.setType(src.getGuestOsId() % 2 == 0 ? "windows" - : "linux"; - dst.bios = new Bios(); - dst.bios.type = "q35_secure_boot"; - dst.type = "desktop"; - dst.origin = "ovirt"; + : "linux"); + dst.setOs(os); + Bios bios = new Bios(); + bios.setType("q35_secure_boot"); + dst.setBios(bios); + dst.setType("desktop"); + dst.setOrigin("ovirt"); + dst.setStateless("false"); if (disksResolver != null) { List diskAttachments = disksResolver.apply(src.getId()); @@ -129,18 +134,18 @@ public static Vm toVm(final UserVmJoinVO src, final Function h dst.setNics(new Nics(nics)); } - dst.actions = new Actions(List.of( - BaseDto.getActionLink("start", dst.href), - BaseDto.getActionLink("stop", dst.href), - BaseDto.getActionLink("shutdown", dst.href) + dst.setActions(new Actions(List.of( + BaseDto.getActionLink("start", dst.getHref()), + BaseDto.getActionLink("stop", dst.getHref()), + BaseDto.getActionLink("shutdown", dst.getHref()) + ))); + dst.setLink(List.of( + BaseDto.getActionLink("diskattachments", dst.getHref()), + BaseDto.getActionLink("nics", dst.getHref()), + BaseDto.getActionLink("reporteddevices", dst.getHref()), + BaseDto.getActionLink("snapshots", dst.getHref()) )); - dst.link = List.of( - BaseDto.getActionLink("diskattachments", dst.href), - BaseDto.getActionLink("nics", dst.href), - BaseDto.getActionLink("reporteddevices", dst.href), - BaseDto.getActionLink("snapshots", dst.href) - ); - dst.tags = new EmptyElement(); + dst.setTags(new EmptyElement()); return dst; } @@ -173,9 +178,6 @@ private static Ref buildRef(final String baseHref, final String suffix, final St if (StringUtils.isBlank(id)) { return null; } - final Ref r = new Ref(); - r.id = id; - r.href = (baseHref != null) ? (baseHref + "/" + suffix + "/" + id) : null; - return r; + return Ref.of((baseHref != null) ? (baseHref + "/" + suffix + "/" + id) : null, id); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java index cf7226227b0c..7d1727d742a0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java @@ -24,8 +24,8 @@ import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.BaseDto; -import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Snapshot; +import org.apache.cloudstack.veeam.api.dto.Vm; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotVO; @@ -36,7 +36,7 @@ public static Snapshot toSnapshot(final VMSnapshotVO vmSnapshotVO, String vmUuid final Snapshot snapshot = new Snapshot(); snapshot.setId(vmSnapshotVO.getUuid()); snapshot.setHref(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmUuid + "/snapshots/" + vmSnapshotVO.getUuid()); - snapshot.setVm(Ref.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmUuid, vmUuid)); + snapshot.setVm(Vm.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vmUuid, vmUuid)); snapshot.setDescription(vmSnapshotVO.getDescription()); snapshot.setSnapshotType("active"); snapshot.setDate(vmSnapshotVO.getCreated().getTime()); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 015b0076334c..1214ccd172af 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -30,6 +30,9 @@ import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.StorageDomain; +import org.apache.cloudstack.veeam.api.dto.StorageDomains; +import org.apache.cloudstack.veeam.api.dto.Vm; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.VolumeJoinVO; @@ -45,22 +48,22 @@ public static Disk toDisk(final VolumeJoinVO vol) { final String diskId = vol.getUuid(); final String diskHref = basePath + DisksRouteHandler.BASE_ROUTE + "/" + diskId; - disk.id = diskId; - disk.href = diskHref; + disk.setId(diskId); + disk.setHref(diskHref); disk.setBootable(String.valueOf(Volume.Type.ROOT.equals(vol.getVolumeType()))); // Names - disk.name = vol.getName(); - disk.alias = vol.getName(); - disk.description = vol.getName(); + disk.setName(vol.getName()); + disk.setAlias(vol.getName()); + disk.setDescription(vol.getName()); // Sizes (bytes) final long size = vol.getSize(); final long actualSize = vol.getVolumeStoreSize(); - disk.provisionedSize = String.valueOf(size); - disk.actualSize = String.valueOf(actualSize); - disk.totalSize = String.valueOf(size); + disk.setProvisionedSize(String.valueOf(size)); + disk.setActualSize(String.valueOf(actualSize)); + disk.setTotalSize(String.valueOf(size)); VolumeStats vs = null; if (List.of(Storage.ImageFormat.VHD, Storage.ImageFormat.QCOW2, Storage.ImageFormat.RAW).contains(vol.getFormat())) { if (vol.getPath() != null) { @@ -72,56 +75,54 @@ public static Disk toDisk(final VolumeJoinVO vol) { } } if (vs != null) { - disk.totalSize = String.valueOf(vs.getVirtualSize()); - disk.actualSize = String.valueOf(vs.getPhysicalSize()); + disk.setTotalSize(String.valueOf(vs.getVirtualSize())); + disk.setActualSize(String.valueOf(vs.getPhysicalSize())); } // Disk format - disk.format = mapFormat(vol.getFormat()); - disk.qcowVersion = "qcow2_v3"; + disk.setFormat(mapFormat(vol.getFormat())); + disk.setQcowVersion("qcow2_v3"); // Content & storage - disk.contentType = "data"; - disk.storageType = "image"; - disk.sparse = "true"; - disk.shareable = "false"; + disk.setContentType("data"); + disk.setStorageType("image"); + disk.setSparse("true"); + disk.setShareable("false"); // Status - disk.status = mapStatus(vol.getState()); + disk.setStatus(mapStatus(vol.getState())); // Backup-related flags (safe defaults) - disk.backup = "none"; - disk.propagateErrors = "false"; - disk.wipeAfterDelete = "false"; + disk.setBackup("none"); + disk.setPropagateErrors("false"); + disk.setWipeAfterDelete("false"); // Image ID (best-effort) - disk.imageId = vol.getPath(); // acceptable placeholder + disk.setImageId(vol.getPath()); // acceptable placeholder // Disk profile (optional) - disk.diskProfile = Ref.of( + disk.setDiskProfile(Ref.of( apiBasePath + "/diskprofiles/" + vol.getDiskOfferingUuid(), String.valueOf(vol.getDiskOfferingUuid()) - ); + )); // Storage domains if (vol.getPoolUuid() != null) { - Disk.StorageDomains sds = new Disk.StorageDomains(); - sds.storageDomain = List.of( - Ref.of( - apiBasePath + "/storagedomains/" + vol.getPoolUuid(), - vol.getPoolUuid() - ) - ); - disk.storageDomains = sds; + StorageDomains sds = new StorageDomains(); + StorageDomain sd = new StorageDomain(); + sd.setHref(apiBasePath + "/storagedomains/" + vol.getPoolUuid()); + sd.setId(vol.getPoolUuid()); + sds.setStorageDomain(List.of(sd)); + disk.setStorageDomains(sds); } // Actions (Veeam checks presence, not behavior) - disk.actions = defaultDiskActions(diskHref); + disk.setActions(defaultDiskActions(diskHref)); // Links - disk.link = List.of( - new Link("disksnapshots", diskHref + "/disksnapshots") - ); + disk.setLink(List.of( + Link.of("disksnapshots", diskHref + "/disksnapshots") + )); return disk; } @@ -137,24 +138,21 @@ public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { final String basePath = VeeamControlService.ContextPath.value(); final String diskAttachmentId = vol.getUuid(); - da.vm = Ref.of( - basePath + VmsRouteHandler.BASE_ROUTE + "/" + vol.getVmUuid(), - vol.getVmUuid() - ); + da.setVm(Vm.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vol.getVmUuid(), vol.getVmUuid())); - da.id = diskAttachmentId; - da.href = da.vm.href + "/diskattachments/" + diskAttachmentId;; + da.setId(diskAttachmentId); + da.setHref(da.getVm().getHref() + "/diskattachments/" + diskAttachmentId);; // Links - da.disk = toDisk(vol); + da.setDisk(toDisk(vol)); // Properties - da.active = "true"; - da.bootable = "false"; - da.iface = "virtio_scsi"; - da.logicalName = vol.getName(); - da.readOnly = "false"; - da.passDiscard = "false"; + da.setActive("true"); + da.setBootable(String.valueOf(Volume.Type.ROOT.equals(vol.getVolumeType()))); + da.setIface("virtio_scsi"); + da.setLogicalName(vol.getName()); + da.setReadOnly("false"); + da.setPassDiscard("false"); return da; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java index 9b4d0d169173..05767e5219d7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java @@ -23,11 +23,19 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Actions { - public List link; + private List link; public Actions() {} public Actions(final List link) { this.link = link; } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java index 7282cc6469b8..93ae93b26d72 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java @@ -21,42 +21,84 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; /** * Root response for GET /ovirt-engine/api */ @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "api") public final class Api { - // repeated @JacksonXmlElementWrapper(useWrapping = false) - public List link; + private List link; + private EmptyElement engineBackup; + private ProductInfo productInfo; + private SpecialObjects specialObjects; + private ApiSummary summary; + private Long time; + private Ref authenticatedUser; + private Ref effectiveUser; - // (empty element) - @JacksonXmlProperty(localName = "engine_backup") - public EmptyElement engineBackup; + public List getLink() { + return link; + } - @JacksonXmlProperty(localName = "product_info") - public ProductInfo productInfo; + public void setLink(List link) { + this.link = link; + } - @JacksonXmlProperty(localName = "special_objects") - public SpecialObjects specialObjects; + public EmptyElement getEngineBackup() { + return engineBackup; + } - @JacksonXmlProperty(localName = "summary") - public ApiSummary summary; + public void setEngineBackup(EmptyElement engineBackup) { + this.engineBackup = engineBackup; + } - // Keep as String to avoid timezone/date parsing friction; you control formatting. - @JacksonXmlProperty(localName = "time") - public Long time; + public ProductInfo getProductInfo() { + return productInfo; + } - @JacksonXmlProperty(localName = "authenticated_user") - public Ref authenticatedUser; + public void setProductInfo(ProductInfo productInfo) { + this.productInfo = productInfo; + } - @JacksonXmlProperty(localName = "effective_user") - public Ref effectiveUser; + public SpecialObjects getSpecialObjects() { + return specialObjects; + } - public Api() {} + public void setSpecialObjects(SpecialObjects specialObjects) { + this.specialObjects = specialObjects; + } + + public ApiSummary getSummary() { + return summary; + } + + public void setSummary(ApiSummary summary) { + this.summary = summary; + } + + public Long getTime() { + return time; + } + + public void setTime(Long time) { + this.time = time; + } + + public Ref getAuthenticatedUser() { + return authenticatedUser; + } + + public void setAuthenticatedUser(Ref authenticatedUser) { + this.authenticatedUser = authenticatedUser; + } + + public Ref getEffectiveUser() { + return effectiveUser; + } + + public void setEffectiveUser(Ref effectiveUser) { + this.effectiveUser = effectiveUser; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java index ba0618f6a9dc..a81c2a1d2745 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java @@ -18,22 +18,44 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class ApiSummary { - @JacksonXmlProperty(localName = "hosts") - public SummaryCount hosts; + private SummaryCount hosts; + private SummaryCount storageDomains; + private SummaryCount users; + private SummaryCount vms; - @JacksonXmlProperty(localName = "storage_domains") - public SummaryCount storageDomains; + public SummaryCount getHosts() { + return hosts; + } - @JacksonXmlProperty(localName = "users") - public SummaryCount users; + public void setHosts(SummaryCount hosts) { + this.hosts = hosts; + } - @JacksonXmlProperty(localName = "vms") - public SummaryCount vms; + public SummaryCount getStorageDomains() { + return storageDomains; + } - public ApiSummary() {} + public void setStorageDomains(SummaryCount storageDomains) { + this.storageDomains = storageDomains; + } + + public SummaryCount getUsers() { + return users; + } + + public void setUsers(SummaryCount users) { + this.users = users; + } + + public SummaryCount getVms() { + return vms; + } + + public void setVms(SummaryCount vms) { + this.vms = vms; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java index 217a16d8131b..6d612fa38ebe 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java @@ -17,20 +17,69 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +public class Backup extends BaseDto { -public class Backup { + private String name; + private String description; + private Long creationDate; + private Vm vm; + private String phase; + private String fromCheckpointId; + private String toCheckpointId; - @JsonProperty("creation_date") - @JacksonXmlProperty(localName = "creation_date") - private String creationDate; + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } - public String getCreationDate() { + public Long getCreationDate() { return creationDate; } - public void setCreationDate(String creationDate) { + public void setCreationDate(Long creationDate) { this.creationDate = creationDate; } + + public Vm getVm() { + return vm; + } + + public void setVm(Vm vm) { + this.vm = vm; + } + + public String getPhase() { + return phase; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + public String getToCheckpointId() { + return toCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java similarity index 67% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java index 39b52c8bd0d2..c1cb39ef5f23 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjectRef.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java @@ -17,22 +17,16 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import java.util.List; -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class SpecialObjectRef { +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - @JacksonXmlProperty(isAttribute = true, localName = "href") - public String href; +public class Backups { - @JacksonXmlProperty(isAttribute = true, localName = "id") - public String id; + @JacksonXmlElementWrapper(useWrapping = false) + public List backup; - public SpecialObjectRef() {} - - public SpecialObjectRef(String href, String id) { - this.href = href; - this.id = id; + public Backups(final List backup) { + this.backup = backup; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java index 013dd9145d90..5ae2eb824224 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java @@ -42,6 +42,6 @@ public void setId(String id) { } public static Link getActionLink(final String action, final String baseHref) { - return new Link(action, baseHref + "/" + action); + return Link.of(action, baseHref + "/" + action); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java index fa9e46ba87cf..ca68bfe475ab 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java @@ -21,13 +21,23 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Bios { - public String type; // "uefi" or "bios" or whatever mapping you choose - public BootMenu bootMenu = new BootMenu(); + private String type; // "uefi" or "bios" or whatever mapping you choose + private BootMenu bootMenu = new BootMenu(); - public Bios() {} + public String getType() { + return type; + } - public Bios(final String type) { + public void setType(String type) { this.type = type; } + + public BootMenu getBootMenu() { + return bootMenu; + } + + public void setBootMenu(BootMenu bootMenu) { + this.bootMenu = bootMenu; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java index 714b256596ad..6a354d5e749d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java @@ -22,5 +22,13 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class BootMenu { - public String enabled = "false"; + private String enabled = "false"; + + public String getEnabled() { + return enabled; + } + + public void setEnabled(String enabled) { + this.enabled = enabled; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java index c90a3ea4c281..7a87bfb09492 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java @@ -18,14 +18,10 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public class Certificate { - @JsonProperty("organization") private String organization; - - @JsonProperty("subject") private String subject; public String getOrganization() { return organization; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoint.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoint.java new file mode 100644 index 000000000000..763875535904 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoint.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +public class Checkpoint extends BaseDto { + + private String name; + private String description; + private String creationDate; + private Vm vm; + private String state; + private String parentId; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreationDate() { + return creationDate; + } + + public void setCreationDate(String creationDate) { + this.creationDate = creationDate; + } + + public Vm getVm() { + return vm; + } + + public void setVm(Vm vm) { + this.vm = vm; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getParentId() { + return parentId; + } + + public void setParentId(String parentId) { + this.parentId = parentId; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java similarity index 54% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java index 1535e0d4727c..7cc346202a91 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OsVersion.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java @@ -17,24 +17,26 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class OsVersion { - @JsonProperty("full_version") - private String fullVersion; - - @JsonProperty("major") - private String major; - - @JsonProperty("minor") - private String minor; - - public String getFullVersion() { return fullVersion; } - public void setFullVersion(String fullVersion) { this.fullVersion = fullVersion; } - public String getMajor() { return major; } - public void setMajor(String major) { this.major = major; } - public String getMinor() { return minor; } - public void setMinor(String minor) { this.minor = minor; } +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; + +public class Checkpoints { + + @JacksonXmlElementWrapper(useWrapping = false) + private List checkpoint; + + public Checkpoints() {} + + public Checkpoints(final List checkpoint) { + this.checkpoint = checkpoint; + } + + public List getCheckpoint() { + return checkpoint; + } + + public void setCheckpoint(List checkpoint) { + this.checkpoint = checkpoint; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java index cdd4a18e2cc9..650177a5e45b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java @@ -20,148 +20,315 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlRootElement(localName = "cluster") -public final class Cluster { +public final class Cluster extends BaseDto { + + private String name; + private String description; + private String comment; + private String ballooningEnabled; + private String biosType; + private Cpu cpu; + private CustomSchedulingPolicyProperties customSchedulingPolicyProperties; + private ErrorHandling errorHandling; + private FencingPolicy fencingPolicy; + private String fipsMode; // "disabled" + private String firewallType; // "firewalld" + private String glusterService; + private String haReservation; + private Ksm ksm; + private String logMaxMemoryUsedThreshold; + private String logMaxMemoryUsedThresholdType; + private MemoryPolicy memoryPolicy; + private Migration migration; + private RequiredRngSources requiredRngSources; + private String switchType; + private String threadsAsCores; + private String trustedService; + private String tunnelMigration; + private String upgradeInProgress; + private String upgradePercentComplete; + private Version version; + private String virtService; + private String vncEncryption; + private Ref dataCenter; + private Ref macPool; + private Ref schedulingPolicy; + private Actions actions; + @JacksonXmlElementWrapper(useWrapping = false) + private List link; - // --- common identity - public String href; - public String id; - public String name; - public String description; - public String comment; + public String getName() { + return name; + } - // --- oVirt-ish knobs (strings in oVirt JSON) - @JsonProperty("ballooning_enabled") - @JacksonXmlProperty(localName = "ballooning_enabled") - public String ballooningEnabled; // "true"/"false" + public void setName(String name) { + this.name = name; + } - @JsonProperty("bios_type") - @JacksonXmlProperty(localName = "bios_type") - public String biosType; // e.g. "q35_ovmf" + public String getDescription() { + return description; + } - public ClusterCpu cpu; + public void setDescription(String description) { + this.description = description; + } - @JsonProperty("custom_scheduling_policy_properties") - @JacksonXmlProperty(localName = "custom_scheduling_policy_properties") - public CustomSchedulingPolicyProperties customSchedulingPolicyProperties; + public String getComment() { + return comment; + } - @JsonProperty("error_handling") - @JacksonXmlProperty(localName = "error_handling") - public ErrorHandling errorHandling; + public void setComment(String comment) { + this.comment = comment; + } - @JsonProperty("fencing_policy") - @JacksonXmlProperty(localName = "fencing_policy") - public FencingPolicy fencingPolicy; + public String getBallooningEnabled() { + return ballooningEnabled; + } - @JsonProperty("fips_mode") - @JacksonXmlProperty(localName = "fips_mode") - public String fipsMode; // "disabled" + public void setBallooningEnabled(String ballooningEnabled) { + this.ballooningEnabled = ballooningEnabled; + } - @JsonProperty("firewall_type") - @JacksonXmlProperty(localName = "firewall_type") - public String firewallType; // "firewalld" + public String getBiosType() { + return biosType; + } - @JsonProperty("gluster_service") - @JacksonXmlProperty(localName = "gluster_service") - public String glusterService; + public void setBiosType(String biosType) { + this.biosType = biosType; + } - @JsonProperty("ha_reservation") - @JacksonXmlProperty(localName = "ha_reservation") - public String haReservation; + public Cpu getCpu() { + return cpu; + } - public Ksm ksm; + public void setCpu(Cpu cpu) { + this.cpu = cpu; + } - @JsonProperty("log_max_memory_used_threshold") - @JacksonXmlProperty(localName = "log_max_memory_used_threshold") - public String logMaxMemoryUsedThreshold; + public CustomSchedulingPolicyProperties getCustomSchedulingPolicyProperties() { + return customSchedulingPolicyProperties; + } - @JsonProperty("log_max_memory_used_threshold_type") - @JacksonXmlProperty(localName = "log_max_memory_used_threshold_type") - public String logMaxMemoryUsedThresholdType; + public void setCustomSchedulingPolicyProperties(CustomSchedulingPolicyProperties customSchedulingPolicyProperties) { + this.customSchedulingPolicyProperties = customSchedulingPolicyProperties; + } - @JsonProperty("memory_policy") - @JacksonXmlProperty(localName = "memory_policy") - public MemoryPolicy memoryPolicy; + public ErrorHandling getErrorHandling() { + return errorHandling; + } - public Migration migration; + public void setErrorHandling(ErrorHandling errorHandling) { + this.errorHandling = errorHandling; + } - @JsonProperty("required_rng_sources") - @JacksonXmlProperty(localName = "required_rng_sources") - public RequiredRngSources requiredRngSources; + public FencingPolicy getFencingPolicy() { + return fencingPolicy; + } - @JsonProperty("switch_type") - @JacksonXmlProperty(localName = "switch_type") - public String switchType; + public void setFencingPolicy(FencingPolicy fencingPolicy) { + this.fencingPolicy = fencingPolicy; + } - @JsonProperty("threads_as_cores") - @JacksonXmlProperty(localName = "threads_as_cores") - public String threadsAsCores; + public String getFipsMode() { + return fipsMode; + } - @JsonProperty("trusted_service") - @JacksonXmlProperty(localName = "trusted_service") - public String trustedService; + public void setFipsMode(String fipsMode) { + this.fipsMode = fipsMode; + } - @JsonProperty("tunnel_migration") - @JacksonXmlProperty(localName = "tunnel_migration") - public String tunnelMigration; + public String getFirewallType() { + return firewallType; + } - @JsonProperty("upgrade_in_progress") - @JacksonXmlProperty(localName = "upgrade_in_progress") - public String upgradeInProgress; + public void setFirewallType(String firewallType) { + this.firewallType = firewallType; + } - @JsonProperty("upgrade_percent_complete") - @JacksonXmlProperty(localName = "upgrade_percent_complete") - public String upgradePercentComplete; + public String getGlusterService() { + return glusterService; + } - public Version version; + public void setGlusterService(String glusterService) { + this.glusterService = glusterService; + } - @JsonProperty("virt_service") - @JacksonXmlProperty(localName = "virt_service") - public String virtService; + public String getHaReservation() { + return haReservation; + } - @JsonProperty("vnc_encryption") - @JacksonXmlProperty(localName = "vnc_encryption") - public String vncEncryption; + public void setHaReservation(String haReservation) { + this.haReservation = haReservation; + } - // --- references - @JsonProperty("data_center") - @JacksonXmlProperty(localName = "data_center") - public Ref dataCenter; + public Ksm getKsm() { + return ksm; + } - @JsonProperty("mac_pool") - @JacksonXmlProperty(localName = "mac_pool") - public Ref macPool; + public void setKsm(Ksm ksm) { + this.ksm = ksm; + } - @JsonProperty("scheduling_policy") - @JacksonXmlProperty(localName = "scheduling_policy") - public Ref schedulingPolicy; + public String getLogMaxMemoryUsedThreshold() { + return logMaxMemoryUsedThreshold; + } - // --- actions + links - public Actions actions; + public void setLogMaxMemoryUsedThreshold(String logMaxMemoryUsedThreshold) { + this.logMaxMemoryUsedThreshold = logMaxMemoryUsedThreshold; + } - @JacksonXmlElementWrapper(useWrapping = false) - public List link; + public String getLogMaxMemoryUsedThresholdType() { + return logMaxMemoryUsedThresholdType; + } - public Cluster() {} + public void setLogMaxMemoryUsedThresholdType(String logMaxMemoryUsedThresholdType) { + this.logMaxMemoryUsedThresholdType = logMaxMemoryUsedThresholdType; + } - // ===== nested DTOs ===== + public MemoryPolicy getMemoryPolicy() { + return memoryPolicy; + } - @JsonInclude(JsonInclude.Include.NON_NULL) - public static final class ClusterCpu { - public String architecture; - public String type; + public void setMemoryPolicy(MemoryPolicy memoryPolicy) { + this.memoryPolicy = memoryPolicy; + } + + public Migration getMigration() { + return migration; + } + + public void setMigration(Migration migration) { + this.migration = migration; + } + + public RequiredRngSources getRequiredRngSources() { + return requiredRngSources; + } + + public void setRequiredRngSources(RequiredRngSources requiredRngSources) { + this.requiredRngSources = requiredRngSources; + } + + public String getSwitchType() { + return switchType; + } + + public void setSwitchType(String switchType) { + this.switchType = switchType; + } + + public String getThreadsAsCores() { + return threadsAsCores; + } + + public void setThreadsAsCores(String threadsAsCores) { + this.threadsAsCores = threadsAsCores; + } + + public String getTrustedService() { + return trustedService; + } + + public void setTrustedService(String trustedService) { + this.trustedService = trustedService; + } + + public String getTunnelMigration() { + return tunnelMigration; + } + + public void setTunnelMigration(String tunnelMigration) { + this.tunnelMigration = tunnelMigration; + } + + public String getUpgradeInProgress() { + return upgradeInProgress; + } + + public void setUpgradeInProgress(String upgradeInProgress) { + this.upgradeInProgress = upgradeInProgress; + } + + public String getUpgradePercentComplete() { + return upgradePercentComplete; + } + + public void setUpgradePercentComplete(String upgradePercentComplete) { + this.upgradePercentComplete = upgradePercentComplete; + } + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } + + public String getVirtService() { + return virtService; + } + + public void setVirtService(String virtService) { + this.virtService = virtService; + } + + public String getVncEncryption() { + return vncEncryption; + } + + public void setVncEncryption(String vncEncryption) { + this.vncEncryption = vncEncryption; + } + + public Ref getDataCenter() { + return dataCenter; + } + + public void setDataCenter(Ref dataCenter) { + this.dataCenter = dataCenter; + } + + public Ref getMacPool() { + return macPool; + } + + public void setMacPool(Ref macPool) { + this.macPool = macPool; + } + + public Ref getSchedulingPolicy() { + return schedulingPolicy; + } + + public void setSchedulingPolicy(Ref schedulingPolicy) { + this.schedulingPolicy = schedulingPolicy; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; } @JsonInclude(JsonInclude.Include.NON_NULL) public static final class CustomSchedulingPolicyProperties { @JacksonXmlElementWrapper(useWrapping = false) - @JsonProperty("property") public List property; } @@ -173,29 +340,15 @@ public static final class Property { @JsonInclude(JsonInclude.Include.NON_NULL) public static final class ErrorHandling { - @JsonProperty("on_error") - @JacksonXmlProperty(localName = "on_error") public String onError; // "migrate" } @JsonInclude(JsonInclude.Include.NON_NULL) public static final class FencingPolicy { public String enabled; - - @JsonProperty("skip_if_connectivity_broken") - @JacksonXmlProperty(localName = "skip_if_connectivity_broken") public SkipIfConnectivityBroken skipIfConnectivityBroken; - - @JsonProperty("skip_if_gluster_bricks_up") - @JacksonXmlProperty(localName = "skip_if_gluster_bricks_up") public String skipIfGlusterBricksUp; - - @JsonProperty("skip_if_gluster_quorum_not_met") - @JacksonXmlProperty(localName = "skip_if_gluster_quorum_not_met") public String skipIfGlusterQuorumNotMet; - - @JsonProperty("skip_if_sd_active") - @JacksonXmlProperty(localName = "skip_if_sd_active") public SkipIfSdActive skipIfSdActive; } @@ -213,20 +366,12 @@ public static final class SkipIfSdActive { @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Ksm { public String enabled; - - @JsonProperty("merge_across_nodes") - @JacksonXmlProperty(localName = "merge_across_nodes") public String mergeAcrossNodes; } @JsonInclude(JsonInclude.Include.NON_NULL) public static final class MemoryPolicy { - @JsonProperty("over_commit") - @JacksonXmlProperty(localName = "over_commit") public OverCommit overCommit; - - @JsonProperty("transparent_hugepages") - @JacksonXmlProperty(localName = "transparent_hugepages") public TransparentHugepages transparentHugepages; } @@ -242,39 +387,22 @@ public static final class TransparentHugepages { @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Migration { - @JsonProperty("auto_converge") - @JacksonXmlProperty(localName = "auto_converge") public String autoConverge; - public Bandwidth bandwidth; - public String compressed; public String encrypted; - - @JsonProperty("parallel_migrations_policy") - @JacksonXmlProperty(localName = "parallel_migrations_policy") public String parallelMigrationsPolicy; - public Ref policy; } @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Bandwidth { - @JsonProperty("assignment_method") - @JacksonXmlProperty(localName = "assignment_method") public String assignmentMethod; } @JsonInclude(JsonInclude.Include.NON_NULL) public static final class RequiredRngSources { - @JsonProperty("required_rng_source") @JacksonXmlElementWrapper(useWrapping = false) public List requiredRngSource; } - - @JsonInclude(JsonInclude.Include.NON_NULL) - public static final class Version { - public String major; - public String minor; - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java index 67eca4c989c2..4755962bd01d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java @@ -22,19 +22,25 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "clusters") public final class Clusters { @JsonProperty("cluster") @JacksonXmlElementWrapper(useWrapping = false) - public List cluster; + private List cluster; public Clusters() {} public Clusters(final List cluster) { this.cluster = cluster; } + + public List getCluster() { + return cluster; + } + + public void setCluster(List cluster) { + this.cluster = cluster; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java index 79c6504a9269..97459b40cd8d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java @@ -18,27 +18,23 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class Cpu { - @JsonProperty("name") private String name; - - @JsonProperty("speed") private Integer speed; - public String architecture; - public Topology topology; - - public Cpu() {} - - public Cpu(final String architecture, final Topology topology) { - this.architecture = architecture; - this.topology = topology; - } + private String architecture; + private String type; + private Topology topology; public String getName() { return name; } public void setName(String name) { this.name = name; } public Integer getSpeed() { return speed; } public void setSpeed(Integer speed) { this.speed = speed; } + public String getArchitecture() { return architecture; } + public void setArchitecture(String architecture) { this.architecture = architecture; } + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public Topology getTopology() { return topology; } + public void setTopology(Topology topology) { this.topology = topology; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java index f0b8a8aff5de..9c3aed49406d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java @@ -20,43 +20,110 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlRootElement(localName = "data_center") -public final class DataCenter { +public final class DataCenter extends BaseDto { + private String local; + private String quotaMode; + private String status; + private String storageFormat; + private SupportedVersions supportedVersions; + private Version version; + private Ref macPool; + private Actions actions; + private String name; + private String description; + @JacksonXmlElementWrapper(useWrapping = false) + public List link; - // keep strings to match oVirt JSON ("false", "disabled", "up", "v5", etc.) - public String local; + public String getLocal() { + return local; + } - @JsonProperty("quota_mode") - public String quotaMode; + public void setLocal(String local) { + this.local = local; + } - public String status; + public String getQuotaMode() { + return quotaMode; + } - @JsonProperty("storage_format") - public String storageFormat; + public void setQuotaMode(String quotaMode) { + this.quotaMode = quotaMode; + } - @JsonProperty("supported_versions") - public SupportedVersions supportedVersions; + public String getStatus() { + return status; + } - public Version version; + public void setStatus(String status) { + this.status = status; + } - @JsonProperty("mac_pool") - public Ref macPool; + public String getStorageFormat() { + return storageFormat; + } - public Actions actions; + public void setStorageFormat(String storageFormat) { + this.storageFormat = storageFormat; + } - public String name; - public String description; + public SupportedVersions getSupportedVersions() { + return supportedVersions; + } - @JacksonXmlElementWrapper(useWrapping = false) - public List link; + public void setSupportedVersions(SupportedVersions supportedVersions) { + this.supportedVersions = supportedVersions; + } + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } + + public Ref getMacPool() { + return macPool; + } + + public void setMacPool(Ref macPool) { + this.macPool = macPool; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } - public String href; - public String id; + public List getLink() { + return link; + } - public DataCenter() {} + public void setLink(List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java index a99363a27135..fa44bbf86fc7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java @@ -20,10 +20,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; /** * Root collection wrapper: @@ -32,11 +29,8 @@ * } */ @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "data_centers") -@JsonPropertyOrder({ "data_center" }) public final class DataCenters { - @JsonProperty("data_center") @JacksonXmlElementWrapper(useWrapping = false) public List dataCenter; @@ -44,4 +38,12 @@ public DataCenters() {} public DataCenters(final List dataCenter) { this.dataCenter = dataCenter; } + + public List getDataCenter() { + return dataCenter; + } + + public void setDataCenter(List dataCenter) { + this.dataCenter = dataCenter; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java index 6ba2f1d736b8..ce609592f154 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java @@ -17,93 +17,231 @@ package org.apache.cloudstack.veeam.api.dto; +import java.util.List; + import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; -import java.util.List; - @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlRootElement(localName = "disk") -public final class Disk { +public final class Disk extends BaseDto { private String bootable; + private String actualSize; + private String alias; + private String backup; + private String contentType; + private String format; + private String imageId; + private String propagateErrors; + private String initialSize; + private String provisionedSize; + private String qcowVersion; + private String shareable; + private String sparse; + private String status; + private String storageType; + private String totalSize; + private String wipeAfterDelete; + private Ref diskProfile; + private Ref quota; + private StorageDomains storageDomains; + private Actions actions; + private String name; + private String description; + @JacksonXmlElementWrapper(useWrapping = false) + private List link; - @JsonProperty("actual_size") - public String actualSize; + public String getBootable() { + return bootable; + } - public String alias; - public String backup; + public void setBootable(String bootable) { + this.bootable = bootable; + } - @JsonProperty("content_type") - public String contentType; + public String getActualSize() { + return actualSize; + } + + public void setActualSize(String actualSize) { + this.actualSize = actualSize; + } + + public String getAlias() { + return alias; + } - public String format; + public void setAlias(String alias) { + this.alias = alias; + } - @JsonProperty("image_id") - public String imageId; + public String getBackup() { + return backup; + } - @JsonProperty("propagate_errors") - public String propagateErrors; + public void setBackup(String backup) { + this.backup = backup; + } + + public String getContentType() { + return contentType; + } - @JsonProperty("initial_size") - public String initialSize; + public void setContentType(String contentType) { + this.contentType = contentType; + } - @JsonProperty("provisioned_size") - public String provisionedSize; + public String getFormat() { + return format; + } - @JsonProperty("qcow_version") - public String qcowVersion; + public void setFormat(String format) { + this.format = format; + } - public String shareable; - public String sparse; - public String status; + public String getImageId() { + return imageId; + } - @JsonProperty("storage_type") - public String storageType; + public void setImageId(String imageId) { + this.imageId = imageId; + } - @JsonProperty("total_size") - public String totalSize; + public String getPropagateErrors() { + return propagateErrors; + } - @JsonProperty("wipe_after_delete") - public String wipeAfterDelete; + public void setPropagateErrors(String propagateErrors) { + this.propagateErrors = propagateErrors; + } - @JsonProperty("disk_profile") - public Ref diskProfile; + public String getInitialSize() { + return initialSize; + } - public Ref quota; + public void setInitialSize(String initialSize) { + this.initialSize = initialSize; + } - @JsonProperty("storage_domains") - public StorageDomains storageDomains; + public String getProvisionedSize() { + return provisionedSize; + } - public Actions actions; + public void setProvisionedSize(String provisionedSize) { + this.provisionedSize = provisionedSize; + } - public String name; - public String description; + public String getQcowVersion() { + return qcowVersion; + } - @JacksonXmlElementWrapper(useWrapping = false) - public List link; + public void setQcowVersion(String qcowVersion) { + this.qcowVersion = qcowVersion; + } - public String href; - public String id; + public String getShareable() { + return shareable; + } - public Disk() {} + public void setShareable(String shareable) { + this.shareable = shareable; + } - public String getBootable() { - return bootable; + public String getSparse() { + return sparse; } - public void setBootable(String bootable) { - this.bootable = bootable; + public void setSparse(String sparse) { + this.sparse = sparse; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStorageType() { + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; + } + + public String getTotalSize() { + return totalSize; + } + + public void setTotalSize(String totalSize) { + this.totalSize = totalSize; + } + + public String getWipeAfterDelete() { + return wipeAfterDelete; + } + + public void setWipeAfterDelete(String wipeAfterDelete) { + this.wipeAfterDelete = wipeAfterDelete; + } + + public Ref getDiskProfile() { + return diskProfile; + } + + public void setDiskProfile(Ref diskProfile) { + this.diskProfile = diskProfile; + } + + public Ref getQuota() { + return quota; + } + + public void setQuota(Ref quota) { + this.quota = quota; + } + + public StorageDomains getStorageDomains() { + return storageDomains; + } + + public void setStorageDomains(StorageDomains storageDomains) { + this.storageDomains = storageDomains; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getLink() { + return link; } - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlRootElement(localName = "storage_domains") - public static final class StorageDomains { - @JsonProperty("storage_domain") - @JacksonXmlElementWrapper(useWrapping = false) - public List storageDomain; - public StorageDomains() {} + public void setLink(List link) { + this.link = link; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java index 578b9462c41d..5b0428efb1b0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java @@ -19,35 +19,92 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "disk_attachment") -public final class DiskAttachment { - - public String active; - public String bootable; +public final class DiskAttachment extends BaseDto { + private String active; + private String bootable; @JsonProperty("interface") - public String iface; // virtio_scsi etc + private String iface; // virtio_scsi etc + private String logicalName; + private String passDiscard; + private String readOnly; + private String usesScsiReservation; + private Disk disk; + private Vm vm; + + public DiskAttachment() {} - @JsonProperty("logical_name") - public String logicalName; + public String getActive() { + return active; + } - @JsonProperty("pass_discard") - public String passDiscard; + public void setActive(String active) { + this.active = active; + } - @JsonProperty("read_only") - public String readOnly; + public String getBootable() { + return bootable; + } - @JsonProperty("uses_scsi_reservation") - public String usesScsiReservation; + public void setBootable(String bootable) { + this.bootable = bootable; + } - public Disk disk; - public Ref vm; + public String getIface() { + return iface; + } - public String href; - public String id; + public void setIface(String iface) { + this.iface = iface; + } - public DiskAttachment() {} + public String getLogicalName() { + return logicalName; + } + + public void setLogicalName(String logicalName) { + this.logicalName = logicalName; + } + + public String getPassDiscard() { + return passDiscard; + } + + public void setPassDiscard(String passDiscard) { + this.passDiscard = passDiscard; + } + + public String getReadOnly() { + return readOnly; + } + + public void setReadOnly(String readOnly) { + this.readOnly = readOnly; + } + + public String getUsesScsiReservation() { + return usesScsiReservation; + } + + public void setUsesScsiReservation(String usesScsiReservation) { + this.usesScsiReservation = usesScsiReservation; + } + + public Disk getDisk() { + return disk; + } + + public void setDisk(Disk disk) { + this.disk = disk; + } + + public Vm getVm() { + return vm; + } + + public void setVm(Vm vm) { + this.vm = vm; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java index deebb9d310aa..827a277ee70a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java @@ -17,24 +17,22 @@ package org.apache.cloudstack.veeam.api.dto; +import java.util.List; + import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -import java.util.List; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "disk_attachments") public final class DiskAttachments { - @JsonProperty("disk_attachment") @JacksonXmlElementWrapper(useWrapping = false) - public List diskAttachment; - - public DiskAttachments() {} + private List diskAttachment; public DiskAttachments(final List diskAttachment) { this.diskAttachment = diskAttachment; } + + public List getDiskAttachment() { + return diskAttachment; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java index 6bb2a705d444..a033d88899a0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java @@ -20,21 +20,19 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "disks") public final class Disks { - @JsonProperty("disk") @JacksonXmlElementWrapper(useWrapping = false) - public List disk; - - public Disks() {} + private List disk; public Disks(final List disk) { this.disk = disk; } + + public List getDisk() { + return disk; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java index 51d4e6eca576..20989d8cbd70 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java @@ -18,18 +18,22 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "fault") public final class Fault { - public String reason; // "Not Found", "Bad Request", "Unauthorized" - public String detail; // full message - - public Fault() {} + private String reason; // "Not Found", "Bad Request", "Unauthorized" + private String detail; // full message public Fault(final String reason, final String detail) { this.reason = reason; this.detail = detail; } + + public String getReason() { + return reason; + } + + public String getDetail() { + return detail; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java index 6f2337418ee6..acddcfd30b15 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java @@ -18,23 +18,13 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public class HardwareInformation { - @JsonProperty("manufacturer") private String manufacturer; - - @JsonProperty("product_name") private String productName; - - @JsonProperty("serial_number") private String serialNumber; - - @JsonProperty("uuid") private String uuid; - - @JsonProperty("version") private String version; public String getManufacturer() { return manufacturer; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java index 5a696d0152d7..5e37b7bf935b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java @@ -17,98 +17,40 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; +import com.fasterxml.jackson.annotation.JsonInclude; + @JsonInclude(JsonInclude.Include.NON_NULL) -public class Host { +public class Host extends BaseDto { - @JsonProperty("address") private String address; - - @JsonProperty("auto_numa_status") private String autoNumaStatus; - - @JsonProperty("certificate") private Certificate certificate; - - @JsonProperty("cpu") private Cpu cpu; - - @JsonProperty("external_status") private String externalStatus; - - @JsonProperty("hardware_information") private HardwareInformation hardwareInformation; - - @JsonProperty("kdump_status") private String kdumpStatus; - - @JsonProperty("libvirt_version") private Version libvirtVersion; - - @JsonProperty("max_scheduling_memory") private String maxSchedulingMemory; - - @JsonProperty("memory") private String memory; - - @JsonProperty("numa_supported") private String numaSupported; - - @JsonProperty("os") private Os os; - - @JsonProperty("port") private String port; - - @JsonProperty("protocol") private String protocol; - - @JsonProperty("reinstallation_required") private String reinstallationRequired; - - @JsonProperty("status") private String status; - - @JsonProperty("summary") private ApiSummary summary; - - @JsonProperty("type") private String type; - - @JsonProperty("update_available") private String updateAvailable; - - @JsonProperty("version") private Version version; - - @JsonProperty("vgpu_placement") private String vgpuPlacement; - - @JsonProperty("cluster") private Ref cluster; - - @JsonProperty("actions") private Actions actions; - - @JsonProperty("name") private String name; - - @JsonProperty("comment") private String comment; - - @JsonProperty("link") private List link; - @JsonProperty("href") - private String href; - - @JsonProperty("id") - private String id; - // getters/setters (generate via IDE) public String getAddress() { return address; } public void setAddress(String address) { this.address = address; } @@ -162,8 +104,4 @@ public class Host { public void setComment(String comment) { this.comment = comment; } public List getLink() { return link; } public void setLink(List link) { this.link = link; } - public String getHref() { return href; } - public void setHref(String href) { this.href = href; } - public String getId() { return id; } - public void setId(String id) { this.id = id; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java index 3a17b79ca05d..f2ff074da5ba 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java @@ -21,41 +21,22 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "image_transfer") -public class ImageTransfer { - - private String id; - private String href; +public class ImageTransfer extends BaseDto { private String active; private String direction; private String format; - - @JsonProperty("inactivity_timeout") private String inactivityTimeout; - private String phase; - - @JsonProperty("proxy_url") private String proxyUrl; - private String shallow; - - @JsonProperty("timeout_policy") private String timeoutPolicy; - - @JsonProperty("transfer_url") private String transferUrl; - private String transferred; - private Backup backup; - private Ref host; private Ref image; private Ref disk; @@ -64,22 +45,6 @@ public class ImageTransfer { @JacksonXmlElementWrapper(useWrapping = false) public List link; - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getHref() { - return href; - } - - public void setHref(String href) { - this.href = href; - } - public String getActive() { return active; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java index 042d45c133d2..43121439b501 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; @JsonInclude(JsonInclude.Include.NON_NULL) -public class Job { +public class Job extends BaseDto { private String autoCleared; private String external; private Long lastUpdated; @@ -33,8 +33,6 @@ public class Job { private Actions actions; private String description; private List link; - private String href; - private String id; // getters and setters public String getAutoCleared() { return autoCleared; } @@ -66,10 +64,4 @@ public class Job { public List getLink() { return link; } public void setLink(List link) { this.link = link; } - - public String getHref() { return href; } - public void setHref(String href) { this.href = href; } - - public String getId() { return id; } - public void setId(String id) { this.id = id; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java index 276cd0a6a5cb..7d67820360f9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java @@ -21,13 +21,29 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Link { - public String rel; - public String href; + private String rel; + private String href; - public Link() {} + public static Link of(final String rel, final String href) { + Link link = new Link(); + link.setRel(rel); + link.setHref(href); + return link; + } + + public String getRel() { + return rel; + } - public Link(final String rel, final String href) { + public void setRel(String rel) { this.rel = rel; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { this.href = href; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java new file mode 100644 index 000000000000..c040323b8d09 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonCreator; + +public class NamedList { + private final String name; + private final List items; + + private NamedList(String name, List items) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("name must be non-empty"); + } + this.name = name; + this.items = items == null ? Collections.emptyList() : items; + } + + public static NamedList of(String name, List items) { + return new NamedList<>(name, items); + } + + @JsonAnyGetter + public Map> asMap() { + return Collections.singletonMap(name, items); + } + + @JsonCreator + public static NamedList fromMap(Map> map) { + if (map == null || map.size() != 1) { + throw new IllegalArgumentException("Expected single-property object for NamedList"); + } + Entry> e = map.entrySet().iterator().next(); + return new NamedList<>(e.getKey(), e.getValue()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java index 0e88914141c0..79e84fb3b172 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -public class Network { +public class Network extends BaseDto { private String mtu; // oVirt prints as string private String portIsolation; // "false" private String stp; // "false" @@ -39,9 +39,6 @@ public class Network { @JsonProperty("link") private List link; - private String href; - private String id; - public Network() {} // ---- getters / setters ---- @@ -75,10 +72,4 @@ public Network() {} public List getLink() { return link; } public void setLink(final List link) { this.link = link; } - - public String getHref() { return href; } - public void setHref(final String href) { this.href = href; } - - public String getId() { return id; } - public void setId(final String id) { this.id = id; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java index dcb9d3505a38..0b0a9043e513 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java @@ -22,10 +22,8 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -public class Nic { +public class Nic extends BaseDto { - private String href; - private String id; private String name; private String description; @JacksonXmlProperty(localName = "interface") @@ -36,28 +34,12 @@ public class Nic { private String plugged; public String synced; private Ref vnicProfile; - private Ref vm; + private Vm vm; private ReportedDevices reportedDevices; public Nic() { } - public String getHref() { - return href; - } - - public void setHref(final String href) { - this.href = href; - } - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - public String getName() { return name; } @@ -122,11 +104,11 @@ public void setVnicProfile(Ref vnicProfile) { this.vnicProfile = vnicProfile; } - public Ref getVm() { + public Vm getVm() { return vm; } - public void setVm(Ref vm) { + public void setVm(Vm vm) { this.vm = vm; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java index e53374e4d103..da73ebd9069b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java @@ -21,11 +21,13 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Os { - public String type; // "rhel_9", "windows_2022", etc. + private String type; - public Os() {} + public String getType() { + return type; + } - public Os(final String type) { + public void setType(String type) { this.type = type; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java index e3618b0e6f9c..7f696a309798 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java @@ -18,19 +18,35 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class ProductInfo { - @JacksonXmlProperty(localName = "instance_id") - public String instanceId; - - @JacksonXmlProperty(localName = "name") + private String instanceId; public String name; - - @JacksonXmlProperty(localName = "version") public Version version; - public ProductInfo() {} + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java index 04ab01f6abdb..4eefbde8ebf6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java @@ -20,20 +20,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; @JsonInclude(JsonInclude.Include.NON_NULL) -public final class Ref { - public String href; - public String id; - public String name; // optional - - public Ref() {} - - public Ref(final String href, final String id, final String name) { - this.href = href; - this.id = id; - this.name = name; - } +public final class Ref extends BaseDto { public static Ref of(final String href, final String id) { - return new Ref(href, id, null); + Ref ref = new Ref(); + ref.setHref(href); + ref.setId(id); + return ref; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java index 14a540699bb0..49011b303dbc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java @@ -17,16 +17,14 @@ package org.apache.cloudstack.veeam.api.dto; -public class ReportedDevice { +public class ReportedDevice extends BaseDto { private String comment; private String description; private Ips ips; - private String id; private Mac Mac; private String name; private String type; - private String href; - private Ref vm; + private Vm vm; public String getComment() { return comment; @@ -52,14 +50,6 @@ public void setIps(Ips ips) { this.ips = ips; } - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - public Mac getMac() { return Mac; } @@ -84,19 +74,11 @@ public void setType(String type) { this.type = type; } - public String getHref() { - return href; - } - - public void setHref(String href) { - this.href = href; - } - - public Ref getVm() { + public Vm getVm() { return vm; } - public void setVm(Ref vm) { + public void setVm(Vm vm) { this.vm = vm; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java index 5f5347e1181d..218a9d227d11 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java @@ -34,7 +34,7 @@ public class Snapshot extends BaseDto { private String description; @JacksonXmlElementWrapper(useWrapping = false) private List link; - private Ref vm; + private Vm vm; public Snapshot() {} @@ -94,11 +94,11 @@ public void setLink(final List link) { this.link = link; } - public Ref getVm() { + public Vm getVm() { return vm; } - public void setVm(Ref vm) { + public void setVm(Vm vm) { this.vm = vm; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java index dc747fa177e1..0ed2297eaad9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java @@ -18,16 +18,26 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class SpecialObjects { - @JacksonXmlProperty(localName = "blank_template") - public SpecialObjectRef blankTemplate; + private Ref blankTemplate; + private Ref rootTag; - @JacksonXmlProperty(localName = "root_tag") - public SpecialObjectRef rootTag; + public Ref getBlankTemplate() { + return blankTemplate; + } - public SpecialObjects() {} + public void setBlankTemplate(Ref blankTemplate) { + this.blankTemplate = blankTemplate; + } + + public Ref getRootTag() { + return rootTag; + } + + public void setRootTag(Ref rootTag) { + this.rootTag = rootTag; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java index edf411ec9be1..4631df35ec67 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java @@ -18,25 +18,53 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class Storage { - public String type; // nfs / glance + private String type; + private String address; + private String path; + private String mountOptions; + private String nfsVersion; - // nfs-ish fields (optional) - public String address; - public String path; + public String getType() { + return type; + } - @JsonProperty("mount_options") - @JacksonXmlProperty(localName = "mount_options") - public String mountOptions; + public void setType(String type) { + this.type = type; + } - @JsonProperty("nfs_version") - @JacksonXmlProperty(localName = "nfs_version") - public String nfsVersion; + public String getAddress() { + return address; + } - public Storage() {} + public void setAddress(String address) { + this.address = address; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + public String getNfsVersion() { + return nfsVersion; + } + + public void setNfsVersion(String nfsVersion) { + this.nfsVersion = nfsVersion; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java index 0b4663fd0395..9dfadd73e0d6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java @@ -20,81 +20,217 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "storage_domain") -public final class StorageDomain { +public final class StorageDomain extends BaseDto { + + private String name; + private String description; + private String comment; + private String available; + private String used; + private String committed; + private String blockSize; + private String warningLowSpaceIndicator; + private String criticalSpaceActionBlocker; + private String status; // e.g. "unattached" (optional in your first object) + private String type; // data / image / iso / export + private String master; // "true"/"false" + private String backup; // "true"/"false" + private String externalStatus; // "ok" + private String storageFormat; // v5 / v1 + private String discardAfterDelete; + private String wipeAfterDelete; + private String supportsDiscard; + private String supportsDiscardZeroesData; + private Storage storage; + private DataCenters dataCenters; + private Actions actions; + @JacksonXmlElementWrapper(useWrapping = false) + private List link; - // Identifiers - public String id; - public String href; + public String getName() { + return name; + } - public String name; - public String description; - public String comment; + public void setName(String name) { + this.name = name; + } - // oVirt returns these as strings in your sample - public String available; - public String used; - public String committed; + public String getDescription() { + return description; + } - @JsonProperty("block_size") - @JacksonXmlProperty(localName = "block_size") - public String blockSize; + public void setDescription(String description) { + this.description = description; + } - @JsonProperty("warning_low_space_indicator") - @JacksonXmlProperty(localName = "warning_low_space_indicator") - public String warningLowSpaceIndicator; + public String getComment() { + return comment; + } - @JsonProperty("critical_space_action_blocker") - @JacksonXmlProperty(localName = "critical_space_action_blocker") - public String criticalSpaceActionBlocker; + public void setComment(String comment) { + this.comment = comment; + } - public String status; // e.g. "unattached" (optional in your first object) - public String type; // data / image / iso / export + public String getAvailable() { + return available; + } - public String master; // "true"/"false" - public String backup; // "true"/"false" + public void setAvailable(String available) { + this.available = available; + } - @JsonProperty("external_status") - @JacksonXmlProperty(localName = "external_status") - public String externalStatus; // "ok" + public String getUsed() { + return used; + } - @JsonProperty("storage_format") - @JacksonXmlProperty(localName = "storage_format") - public String storageFormat; // v5 / v1 + public void setUsed(String used) { + this.used = used; + } - @JsonProperty("discard_after_delete") - @JacksonXmlProperty(localName = "discard_after_delete") - public String discardAfterDelete; + public String getCommitted() { + return committed; + } - @JsonProperty("wipe_after_delete") - @JacksonXmlProperty(localName = "wipe_after_delete") - public String wipeAfterDelete; + public void setCommitted(String committed) { + this.committed = committed; + } - @JsonProperty("supports_discard") - @JacksonXmlProperty(localName = "supports_discard") - public String supportsDiscard; + public String getBlockSize() { + return blockSize; + } - @JsonProperty("supports_discard_zeroes_data") - @JacksonXmlProperty(localName = "supports_discard_zeroes_data") - public String supportsDiscardZeroesData; + public void setBlockSize(String blockSize) { + this.blockSize = blockSize; + } - // Nested - public Storage storage; + public String getWarningLowSpaceIndicator() { + return warningLowSpaceIndicator; + } - @JsonProperty("data_centers") - @JacksonXmlProperty(localName = "data_centers") - public DataCenters dataCenters; + public void setWarningLowSpaceIndicator(String warningLowSpaceIndicator) { + this.warningLowSpaceIndicator = warningLowSpaceIndicator; + } - public Actions actions; + public String getCriticalSpaceActionBlocker() { + return criticalSpaceActionBlocker; + } - @JacksonXmlElementWrapper(useWrapping = false) - public List link; + public void setCriticalSpaceActionBlocker(String criticalSpaceActionBlocker) { + this.criticalSpaceActionBlocker = criticalSpaceActionBlocker; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getMaster() { + return master; + } + + public void setMaster(String master) { + this.master = master; + } + + public String getBackup() { + return backup; + } + + public void setBackup(String backup) { + this.backup = backup; + } + + public String getExternalStatus() { + return externalStatus; + } + + public void setExternalStatus(String externalStatus) { + this.externalStatus = externalStatus; + } + + public String getStorageFormat() { + return storageFormat; + } + + public void setStorageFormat(String storageFormat) { + this.storageFormat = storageFormat; + } + + public String getDiscardAfterDelete() { + return discardAfterDelete; + } + + public void setDiscardAfterDelete(String discardAfterDelete) { + this.discardAfterDelete = discardAfterDelete; + } + + public String getWipeAfterDelete() { + return wipeAfterDelete; + } + + public void setWipeAfterDelete(String wipeAfterDelete) { + this.wipeAfterDelete = wipeAfterDelete; + } + + public String getSupportsDiscard() { + return supportsDiscard; + } + + public void setSupportsDiscard(String supportsDiscard) { + this.supportsDiscard = supportsDiscard; + } + + public String getSupportsDiscardZeroesData() { + return supportsDiscardZeroesData; + } + + public void setSupportsDiscardZeroesData(String supportsDiscardZeroesData) { + this.supportsDiscardZeroesData = supportsDiscardZeroesData; + } + + public Storage getStorage() { + return storage; + } + + public void setStorage(Storage storage) { + this.storage = storage; + } + + public DataCenters getDataCenters() { + return dataCenters; + } + + public void setDataCenters(DataCenters dataCenters) { + this.dataCenters = dataCenters; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } + + public List getLink() { + return link; + } - public StorageDomain() {} + public void setLink(List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java index c2983bf18628..644986998c40 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java @@ -30,10 +30,13 @@ public final class StorageDomains { @JsonProperty("storage_domain") @JacksonXmlElementWrapper(useWrapping = false) - public List storageDomain; + private List storageDomain; - public StorageDomains() {} - public StorageDomains(List storageDomain) { + public List getStorageDomain() { + return storageDomain; + } + + public void setStorageDomain(List storageDomain) { this.storageDomain = storageDomain; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java index a0266a2b89a7..280704f9b51c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java @@ -18,21 +18,23 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class SummaryCount { - @JacksonXmlProperty(localName = "active") - public Integer active; - - @JacksonXmlProperty(localName = "total") - public Integer total; - - public SummaryCount() {} + private Integer active; + private Integer total; public SummaryCount(Integer active, Integer total) { this.active = active; this.total = total; } + + public Integer getActive() { + return active; + } + + public Integer getTotal() { + return total; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java index 7c73b9e5d949..26cfff65620e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java @@ -20,18 +20,19 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @JsonInclude(JsonInclude.Include.NON_NULL) public final class SupportedVersions { - @JsonProperty("version") @JacksonXmlElementWrapper(useWrapping = false) - public List version; + private List version; - public SupportedVersions() {} public SupportedVersions(final List version) { this.version = version; } + + public List getVersion() { + return version; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java index 3458b2cb17f2..564df5b53048 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java @@ -25,11 +25,36 @@ public final class Topology { public Integer cores; public Integer threads; - public Topology() {} + public Topology() { + } public Topology(final Integer sockets, final Integer cores, final Integer threads) { this.sockets = sockets; this.cores = cores; this.threads = threads; } + + public Integer getSockets() { + return sockets; + } + + public void setSockets(Integer sockets) { + this.sockets = sockets; + } + + public Integer getCores() { + return cores; + } + + public void setCores(Integer cores) { + this.cores = cores; + } + + public Integer getThreads() { + return threads; + } + + public void setThreads(Integer threads) { + this.threads = threads; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index cd4601838d1a..4e779e7ff317 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -18,25 +18,55 @@ package org.apache.cloudstack.veeam.api.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @JsonInclude(JsonInclude.Include.NON_NULL) public final class Version { - @JacksonXmlProperty(localName = "build") - public String build; + private String build; + private String fullVersion; + private Integer major; + private Integer minor; + private Integer revision; - @JacksonXmlProperty(localName = "full_version") - public String fullVersion; + public Version() {} - @JacksonXmlProperty(localName = "major") - public Integer major; + public String getBuild() { + return build; + } - @JacksonXmlProperty(localName = "minor") - public Integer minor; + public void setBuild(String build) { + this.build = build; + } - @JacksonXmlProperty(localName = "revision") - public Integer revision; + public String getFullVersion() { + return fullVersion; + } - public Version() {} + public void setFullVersion(String fullVersion) { + this.fullVersion = fullVersion; + } + + public Integer getMajor() { + return major; + } + + public void setMajor(Integer major) { + this.major = major; + } + + public Integer getMinor() { + return minor; + } + + public void setMinor(Integer minor) { + this.minor = minor; + } + + public Integer getRevision() { + return revision; + } + + public void setRevision(Integer revision) { + this.revision = revision; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 2438109105fc..5c6fdf21a1f6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -20,9 +20,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; /** @@ -31,53 +29,64 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlRootElement(localName = "vm") -public final class Vm { - public String href; - public String id; - public String name; - public String description; - - public String status; // "up", "down", ... - - @JsonProperty("stop_reason") - @JacksonXmlProperty(localName = "stop_reason") - public String stopReason; // empty string allowed - +public final class Vm extends BaseDto { + private String name; + private String description; + private String status; // "up", "down", ... + private String stopReason; // empty string allowed private Long creationTime; - - @JsonProperty("stop_time") - @JacksonXmlProperty(localName = "stop_time") - public Long stopTime; // epoch millis + private Long stopTime; // epoch millis private Long startTime; // epoch millis + private Ref template; + private Ref originalTemplate; + private Ref cluster; + private Ref host; + private String memory; // bytes + private Cpu cpu; + private Os os; + private Bios bios; + private String stateless; // true|false + private String type; // "server" + private String origin; // "ovirt" + private Actions actions; // actions.link[] + @JacksonXmlElementWrapper(useWrapping = false) + private List link; // related resources + private EmptyElement tags; // empty + private DiskAttachments diskAttachments; + private Nics nics; + private VmInitialization initialization; - public Ref template; + public String getName() { + return name; + } - @JsonProperty("original_template") - @JacksonXmlProperty(localName = "original_template") - public Ref originalTemplate; + public void setName(String name) { + this.name = name; + } - public Ref cluster; - public Ref host; + public String getDescription() { + return description; + } - public String memory; // bytes - public Cpu cpu; - public Os os; - public Bios bios; + public void setDescription(String description) { + this.description = description; + } - public String stateless = "false"; // true|false - public String type; // "server" - public String origin; // "ovirt" + public String getStatus() { + return status; + } - public Actions actions; // actions.link[] - @JacksonXmlElementWrapper(useWrapping = false) - public List link; // related resources - public EmptyElement tags; // empty - private DiskAttachments diskAttachments; - private Nics nics; + public void setStatus(String status) { + this.status = status; + } - private VmInitialization initialization; + public String getStopReason() { + return stopReason; + } - public Vm() {} + public void setStopReason(String stopReason) { + this.stopReason = stopReason; + } public Long getCreationTime() { return creationTime; @@ -87,6 +96,14 @@ public void setCreationTime(Long creationTime) { this.creationTime = creationTime; } + public Long getStopTime() { + return stopTime; + } + + public void setStopTime(Long stopTime) { + this.stopTime = stopTime; + } + public Long getStartTime() { return startTime; } @@ -95,6 +112,118 @@ public void setStartTime(Long startTime) { this.startTime = startTime; } + public Ref getTemplate() { + return template; + } + + public void setTemplate(Ref template) { + this.template = template; + } + + public Ref getOriginalTemplate() { + return originalTemplate; + } + + public void setOriginalTemplate(Ref originalTemplate) { + this.originalTemplate = originalTemplate; + } + + public Ref getCluster() { + return cluster; + } + + public void setCluster(Ref cluster) { + this.cluster = cluster; + } + + public Ref getHost() { + return host; + } + + public void setHost(Ref host) { + this.host = host; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public Cpu getCpu() { + return cpu; + } + + public void setCpu(Cpu cpu) { + this.cpu = cpu; + } + + public Os getOs() { + return os; + } + + public void setOs(Os os) { + this.os = os; + } + + public Bios getBios() { + return bios; + } + + public void setBios(Bios bios) { + this.bios = bios; + } + + public String getStateless() { + return stateless; + } + + public void setStateless(String stateless) { + this.stateless = stateless; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getOrigin() { + return origin; + } + + public void setOrigin(String origin) { + this.origin = origin; + } + + public Actions getActions() { + return actions; + } + + public void setActions(Actions actions) { + this.actions = actions; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } + + public EmptyElement getTags() { + return tags; + } + + public void setTags(EmptyElement tags) { + this.tags = tags; + } + public DiskAttachments getDiskAttachments() { return diskAttachments; } @@ -118,4 +247,11 @@ public VmInitialization getInitialization() { public void setInitialization(VmInitialization initialization) { this.initialization = initialization; } + + public static Vm of(String href, String id) { + Vm vm = new Vm(); + vm.setHref(href); + vm.setId(id); + return vm; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java index a550b41090b5..efc42ed1c88a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java @@ -26,10 +26,8 @@ * Every vNIC profile MUST reference exactly one network. */ @JsonInclude(JsonInclude.Include.NON_NULL) -public class VnicProfile { +public class VnicProfile extends BaseDto { - private String href; - private String id; private String name; private String description; @@ -41,22 +39,6 @@ public class VnicProfile { public VnicProfile() { } - public String getHref() { - return href; - } - - public void setHref(final String href) { - this.href = href; - } - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - public String getName() { return name; } From 4853453930568f0272a3da64051077c9d322ce56 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Feb 2026 18:12:01 +0530 Subject: [PATCH 035/129] kvm hosts and clusters only Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/dc/dao/ClusterDao.java | 2 ++ .../main/java/com/cloud/dc/dao/ClusterDaoImpl.java | 7 +++++++ .../cloudstack/veeam/adapter/ServerAdapter.java | 2 +- .../java/com/cloud/api/query/dao/HostJoinDao.java | 3 +++ .../com/cloud/api/query/dao/HostJoinDaoImpl.java | 12 ++++++++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 6cfd2608f5de..7952147490ee 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -61,4 +61,6 @@ public interface ClusterDao extends GenericDao { List listDistinctStorageAccessGroups(String name, String keyword); List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, HypervisorType hypervisorType, CPU.CPUArch arch); + + List listByHypervisorType(HypervisorType hypervisorType); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index c63af0a237ba..8988522fc963 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -413,4 +413,11 @@ public List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, Hypervi } return customSearch(sc, null); } + + @Override + public List listByHypervisorType(HypervisorType hypervisorType) { + SearchCriteria sc = ZoneHyTypeSearch.create(); + sc.setParameters("hypervisorType", hypervisorType.toString()); + return listBy(sc); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index d8efa2edbd75..6ccb2c224eac 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -349,7 +349,7 @@ public List listNetworksByDcId(final String uuid) { } public List listAllClusters() { - final List clusters = clusterDao.listAll(); + final List clusters = clusterDao.listByHypervisorType(Hypervisor.HypervisorType.KVM); return ClusterVOToClusterConverter.toClusterList(clusters, this::getZoneById); } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java index bc6ec7931366..005e324cd710 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java @@ -25,6 +25,7 @@ import com.cloud.api.query.vo.HostJoinVO; import com.cloud.host.Host; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.db.GenericDao; public interface HostJoinDao extends GenericDao { @@ -41,4 +42,6 @@ public interface HostJoinDao extends GenericDao { List findByClusterId(Long clusterId, Host.Type type); + List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType); + } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index e7265a7e3b9a..be3598f9cc20 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -413,4 +413,16 @@ private String calculateResourceAllocatedPercentage(float resource, float resour return decimalFormat.format(((float)resource / resourceWithOverProvision * 100.0f)) + "%"; } + @Override + public List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType) { + SearchBuilder sb = createSearchBuilder(); + sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.done(); + + SearchCriteria sc = sb.create(); + sc.setParameters("type", Host.Type.Routing); + sc.setParameters("hypervisorType", hypervisorType); + return listBy(sc); + } } From a9c0215f4bf068d23ab85c786147ab6766a7c7f6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Feb 2026 18:49:08 +0530 Subject: [PATCH 036/129] oauth fix Signed-off-by: Abhishek Kumar --- .../veeam/filter/BearerOrBasicAuthFilter.java | 68 +++++++++---------- .../cloudstack/veeam/sso/SsoService.java | 3 + 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java index 62b6f319b311..511e89ec68c7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.util.Base64; import java.util.List; +import java.util.Map; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -36,6 +37,10 @@ import org.apache.cloudstack.veeam.VeeamControlService; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + public class BearerOrBasicAuthFilter implements Filter { // Keep these aligned with SsoService (move to ConfigKeys later) @@ -43,6 +48,8 @@ public class BearerOrBasicAuthFilter implements Filter { public static final String ISSUER = "veeam-control"; public static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + @Override public void init(FilterConfig filterConfig) {} @Override public void destroy() {} @@ -136,20 +143,35 @@ private boolean verifyJwtHs256(String token) { if (!constantTimeEquals(expectedSig, providedSig)) return false; - final String payloadJson; + Map payloadMap; try { - payloadJson = new String(Base64.getUrlDecoder().decode(payloadB64), StandardCharsets.UTF_8); - } catch (IllegalArgumentException e) { + String payloadJson = new String(Base64.getUrlDecoder().decode(payloadB64), StandardCharsets.UTF_8); + payloadMap = JSON_MAPPER.readValue( + payloadJson, + new TypeReference<>() {} + ); + } catch (IllegalArgumentException | JsonProcessingException e) { return false; } - // Super small “claims” extraction (good enough for our minting format) - final String iss = JsonMini.getString(payloadJson, "iss"); - final String scope = JsonMini.getString(payloadJson, "scope"); - final Long exp = JsonMini.getLong(payloadJson, "exp"); + final String iss = (String)payloadMap.get("iss"); + final String scope = (String)payloadMap.get("scope"); + final Object expObj = payloadMap.get("exp"); + Long exp = null; + if (expObj instanceof Number) { + exp = ((Number) expObj).longValue(); + } else if (expObj instanceof String) { + try { + exp = Long.parseLong((String) expObj); + } catch (NumberFormatException ignored) {} + } - if (!ISSUER.equals(iss)) return false; - if (exp == null || Instant.now().getEpochSecond() >= exp) return false; + if (!ISSUER.equals(iss)) { + return false; + } + if (exp == null || Instant.now().getEpochSecond() >= exp) { + return false; + } return scope != null && hasRequiredScopes(scope); } @@ -216,32 +238,4 @@ private static boolean constantTimeEquals(byte[] x, byte[] y) { for (int i = 0; i < x.length; i++) r |= x[i] ^ y[i]; return r == 0; } - - // Tiny JSON extractor for flat string/number claims. Good enough for tokens you mint. - static final class JsonMini { - static String getString(String json, String key) { - final String needle = "\"" + key + "\":"; - int i = json.indexOf(needle); - if (i < 0) return null; - i += needle.length(); - while (i < json.length() && Character.isWhitespace(json.charAt(i))) i++; - if (i >= json.length() || json.charAt(i) != '"') return null; - i++; - int j = json.indexOf('"', i); - if (j < 0) return null; - return json.substring(i, j); - } - - static Long getLong(String json, String key) { - final String needle = "\"" + key + "\":"; - int i = json.indexOf(needle); - if (i < 0) return null; - i += needle.length(); - while (i < json.length() && Character.isWhitespace(json.charAt(i))) i++; - int j = i; - while (j < json.length() && (Character.isDigit(json.charAt(j)))) j++; - if (j == i) return null; - return Long.parseLong(json.substring(i, j)); - } - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java index 26a29d6d5317..a402b88ab76c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -101,6 +101,8 @@ protected void handleToken(HttpServletRequest req, HttpServletResponse resp, final String effectiveScope = (scope == null) ? "ovirt-app-api" : scope; final long ttl = DEFAULT_TTL_SECONDS; + long nowMillis = Instant.now().toEpochMilli(); + long expMillis = nowMillis + ttl * 1000L; final String token; try { token = JwtUtil.issueHs256Jwt(BearerOrBasicAuthFilter.ISSUER, username, effectiveScope, ttl, @@ -115,6 +117,7 @@ protected void handleToken(HttpServletRequest req, HttpServletResponse resp, payload.put("access_token", token); payload.put("token_type", "bearer"); payload.put("expires_in", ttl); + payload.put("exp", expMillis); payload.put("scope", effectiveScope); io.getWriter().write(resp, HttpServletResponse.SC_OK, payload, outFormat); From 894eef1210a728b18982c268461df43c962941b3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Feb 2026 18:49:37 +0530 Subject: [PATCH 037/129] fix numbers in response Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/api/ApiService.java | 6 +++--- .../converter/ClusterVOToClusterConverter.java | 4 ++-- .../DataCenterJoinVOToDataCenterConverter.java | 4 ++-- .../converter/HostJoinVOToHostConverter.java | 8 +++++--- .../converter/UserVmJoinVOToVmConverter.java | 2 +- .../cloudstack/veeam/api/dto/Version.java | 18 +++++++++--------- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index fbe666882df2..c9024633680d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -105,9 +105,9 @@ private static Api createDummyApi(String basePath) { Version version = new Version(); version.setBuild("8"); version.setFullVersion("4.5.8-0.master.fake.el9"); - version.setMajor(4); - version.setMinor(5); - version.setRevision(0); + version.setMajor("4"); + version.setMinor("5"); + version.setRevision("0"); productInfo.version = version; api.setProductInfo(productInfo); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java index 44789f694bdb..c6a43068562e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -76,8 +76,8 @@ public static Cluster toCluster(final ClusterVO vo, final Function oVirt-ish up/down - if (Arrays.asList(VirtualMachine.State.Running, VirtualMachine.State.Starting, + if (Arrays.asList(VirtualMachine.State.Running, VirtualMachine.State.Migrating, VirtualMachine.State.Restoring).contains(state)) { return "up"; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index 4e779e7ff317..04ba3f99eda9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -24,9 +24,9 @@ public final class Version { private String build; private String fullVersion; - private Integer major; - private Integer minor; - private Integer revision; + private String major; + private String minor; + private String revision; public Version() {} @@ -46,27 +46,27 @@ public void setFullVersion(String fullVersion) { this.fullVersion = fullVersion; } - public Integer getMajor() { + public String getMajor() { return major; } - public void setMajor(Integer major) { + public void setMajor(String major) { this.major = major; } - public Integer getMinor() { + public String getMinor() { return minor; } - public void setMinor(Integer minor) { + public void setMinor(String minor) { this.minor = minor; } - public Integer getRevision() { + public String getRevision() { return revision; } - public void setRevision(Integer revision) { + public void setRevision(String revision) { this.revision = revision; } } From aa7d4bc5905fa29bfbc0a0f5e1ec087d61039592 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Feb 2026 13:36:13 +0530 Subject: [PATCH 038/129] changes for backup job fix Signed-off-by: Abhishek Kumar --- .../admin/backup/FinalizeBackupCmd.java | 8 ++ .../command/admin/backup/StartBackupCmd.java | 65 +++++++++- .../org/apache/cloudstack/backup/Backup.java | 2 +- .../backup/IncrementalBackupService.java | 8 +- .../apache/cloudstack/veeam/RouteHandler.java | 8 ++ .../veeam/adapter/ServerAdapter.java | 111 ++++++++++++++---- .../veeam/api/DisksRouteHandler.java | 2 +- .../veeam/api/ImageTransfersRouteHandler.java | 3 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 27 ++--- .../converter/BackupVOToBackupConverter.java | 42 +++++-- .../VolumeJoinVOToDiskConverter.java | 17 +++ .../backup/IncrementalBackupServiceImpl.java | 66 +++++++---- 12 files changed, 275 insertions(+), 84 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java index 129c570f7acc..3ea69b66b5b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -63,6 +63,14 @@ public Long getBackupId() { return backupId; } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setBackupId(Long backupId) { + this.backupId = backupId; + } + @Override public void execute() { boolean result = incrementalBackupService.finalizeBackup(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java index ea8995801849..b3a87178d164 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -22,24 +22,33 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.context.CallContext; +import com.cloud.event.EventTypes; + @APICommand(name = "startBackup", description = "Start a VM backup session (oVirt-style incremental backup)", responseObject = BackupResponse.class, since = "4.22.0", authorized = {RoleType.Admin}) -public class StartBackupCmd extends BaseCmd implements AdminCmd { + public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { @Inject private IncrementalBackupService incrementalBackupService; + @Inject + private BackupManager backupManager; + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, @@ -47,19 +56,65 @@ public class StartBackupCmd extends BaseCmd implements AdminCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, + type = CommandType.STRING, + description = "the description for the backup") + private String description; + public Long getVmId() { return vmId; } + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + @Override public void execute() { - BackupResponse response = incrementalBackupService.startBackup(this); - response.setResponseName(getCommandName()); - setResponseObject(response); + try { + Backup backup = incrementalBackupService.startBackup(this); + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } } @Override public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + @Override + public void create() { + Backup backup = incrementalBackupService.createBackup(this); + + if (backup != null) { + setEntityId(backup.getId()); + setEntityUuid(backup.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Starting backup for Instance " + vmId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 014fc3c483b0..bc464beeb6d8 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -41,7 +41,7 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Integer getNbdPort(); enum Status { - Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged + Allocated, Queued, BackingUp, ReadyForTransfer, FinalizingTransfer, BackedUp, Error, Failed, Restoring, Removed, Expunged } class Metric { diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index 67ef7175c416..ed97f780db1c 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -26,7 +26,6 @@ import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.CheckpointResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -44,11 +43,16 @@ public interface IncrementalBackupService extends Configurable, PluggableService "10", "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); + /** + * Creates a backup session for a VM + */ + Backup createBackup(StartBackupCmd cmd); + /** * Start a backup session for a VM * Creates a new checkpoint and starts NBD server for pull-mode backup */ - BackupResponse startBackup(StartBackupCmd cmd); + Backup startBackup(StartBackupCmd cmd); /** * Finalize a backup session diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index a955eeac0203..4e0381be6992 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.logging.log4j.Logger; import com.cloud.utils.component.Adapter; @@ -43,6 +44,12 @@ default String getSanitizedPath(String path) { return path; } + static String getRequestData(HttpServletRequest req, Logger logger) { + String data = RouteHandler.getRequestData(req); + logger.info("Received method: {} request. Request-data: {}", req.getMethod(), data); + return data; + } + static String getRequestData(HttpServletRequest req) { String contentType = req.getContentType(); if (contentType == null) { @@ -52,6 +59,7 @@ static String getRequestData(HttpServletRequest req) { if (!"application/json".equals(mime) && !"application/x-www-form-urlencoded".equals(mime)) { return null; } + String result = null; try { StringBuilder data = new StringBuilder(); String line; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 6ccb2c224eac..761abb3f0ab5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; @@ -37,8 +38,8 @@ import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; -import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; @@ -66,6 +67,9 @@ import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -102,6 +106,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import com.cloud.api.query.dao.AsyncJobJoinDao; import com.cloud.api.query.dao.DataCenterJoinDao; @@ -246,6 +251,9 @@ public class ServerAdapter extends ManagerBase { @Inject ApiServerService apiServerService; + @Inject + AsyncJobDao asyncJobDao; + @Inject AsyncJobJoinDao asyncJobJoinDao; @@ -840,7 +848,7 @@ public List listAllImageTransfers() { } public ImageTransfer getImageTransfer(String uuid) { - ImageTransferVO vo = imageTransferDao.findByUuid(uuid); + ImageTransferVO vo = imageTransferDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } @@ -863,7 +871,15 @@ public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { throw new InvalidParameterValueException("Invalid or missing direction"); } Format format = EnumUtils.fromString(Format.class, request.getFormat()); - return createImageTransfer(null, volumeVO.getId(), direction, format); + Long backupId = null; + if (request.getBackup() != null && StringUtils.isNotBlank(request.getBackup().getId())) { + BackupVO backupVO = backupDao.findByUuid(request.getBackup().getId()); + if (backupVO == null) { + throw new InvalidParameterValueException("Backup with ID " + request.getBackup().getId() + " not found"); + } + backupId = backupVO.getId(); + } + return createImageTransfer(backupId, volumeVO.getId(), direction, format); } public boolean handleCancelImageTransfer(String uuid) { @@ -887,7 +903,7 @@ private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Directio CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = - incrementalBackupService.createImageTransfer(volumeId, null, direction, format); + incrementalBackupService.createImageTransfer(volumeId, backupId, direction, format); ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); } finally { @@ -1054,7 +1070,7 @@ public List listBackupsByInstanceUuid(final String uuid) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } List backups = backupDao.searchByVmIds(List.of(vo.getId())); - return BackupVOToBackupConverter.toBackupList(backups, id -> vo); + return BackupVOToBackupConverter.toBackupList(backups, id -> vo, this::getHostById); } public Backup createInstanceBackup(final String vmUuid, final Backup request) { @@ -1062,26 +1078,26 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + // Register a context as resource owner + Account account = accountService.getAccount(vmVo.getAccountId()); + CallContext ctx = CallContext.register(vmVo.getUserId(), vmVo.getAccountId()); try { - CreateBackupCmd cmd = new CreateBackupCmd(); + StartBackupCmd cmd = new StartBackupCmd(); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); params.put(ApiConstants.NAME, request.getName()); params.put(ApiConstants.DESCRIPTION, request.getDescription()); ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + apiServerService.processAsyncCmd(cmd, params, ctx, vmVo.getUserId(), account); if (result.objectId == null) { - throw new CloudRuntimeException("No backup ID returned"); + throw new CloudRuntimeException("Unexpected backup ID returned"); } BackupVO vo = backupDao.findById(result.objectId); if (vo == null) { throw new CloudRuntimeException("Backup not found"); } - return BackupVOToBackupConverter.toBackup(vo, id -> vmVo); + return BackupVOToBackupConverter.toBackup(vo, id -> vmVo, this::getHostById, this::getBackupDisks); } catch (Exception e) { throw new CloudRuntimeException("Failed to create backup: " + e.getMessage(), e); } finally { @@ -1089,16 +1105,53 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { } } + @Nullable + private BackupVO getBackupFromJob(ApiServerService.AsyncCmdResult result, UserVmVO vmVo) { + AsyncJobVO jobVo = null; + // wait for job to complete and get backup ID + long timeoutNanos = TimeUnit.MINUTES.toNanos(2); + final long deadline = System.nanoTime() + timeoutNanos; + long sleepMillis = 1000; + while (System.nanoTime() < deadline) { + jobVo = asyncJobDao.findByIdIncludingRemoved(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for backup creation"); + } + if (!JobInfo.Status.IN_PROGRESS.equals(jobVo.getStatus())) { + break; + } + try { + Thread.sleep(sleepMillis); + // back off gradually to reduce DB pressure + sleepMillis = Math.min(5000, sleepMillis + 500); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new CloudRuntimeException("Interrupted while waiting for backup creation job", ie); + } + } + // if still in progress after timeout, fail fast + if (jobVo != null && JobInfo.Status.IN_PROGRESS.equals(jobVo.getStatus())) { + throw new CloudRuntimeException("Timed out waiting for backup creation job"); + } + BackupVO vo = null; + List backups = backupDao.searchByVmIds(List.of(vmVo.getId())); + if (CollectionUtils.isNotEmpty(backups)) { + vo = backups.get(0); + } + return vo; + } + public Backup getBackup(String uuid) { BackupVO vo = backupDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); } - return BackupVOToBackupConverter.toBackup(vo, id -> userVmDao.findById(id)); + return BackupVOToBackupConverter.toBackup(vo, id -> userVmDao.findById(id), this::getHostById, + this::getBackupDisks); } public List listDisksByBackupUuid(final String uuid) { - throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implmenented"); + throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implemented"); // BackupVO vo = backupDao.findByUuid(uuid); // if (vo == null) { // throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); @@ -1106,28 +1159,28 @@ public List listDisksByBackupUuid(final String uuid) { // return VolumeJoinVOToDiskConverter.toDiskList(volumes); } - public void finalizeBackup(final String vmUuid, final String uuid, String data) { - ResourceAction action = null; - UserVmVO vmVo = userVmDao.findByUuid(vmUuid); - if (vmVo == null) { + public Backup finalizeBackup(final String vmUuid, final String backupUuid) { + UserVmVO vm = userVmDao.findByUuid(vmUuid); + if (vm == null) { throw new InvalidParameterValueException("Instance with ID " + vmUuid + " not found"); } - BackupVO vo = backupDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); + BackupVO backup = backupDao.findByUuid(backupUuid); + if (backup == null) { + throw new InvalidParameterValueException("Backup with ID " + backupUuid + " not found"); } Pair serviceUserAccount = createServiceAccountIfNeeded(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { FinalizeBackupCmd cmd = new FinalizeBackupCmd(); ComponentContext.inject(cmd); - Map params = new HashMap<>(); - params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); - params.put(ApiConstants.BACKUP_ID, vo.getUuid()); + cmd.setBackupId(backup.getId()); + cmd.setVmId(vm.getId()); boolean result = incrementalBackupService.finalizeBackup(cmd); if (!result) { throw new CloudRuntimeException("Failed to finalize backup"); } + backup = backupDao.findById(backup.getId()); + return BackupVOToBackupConverter.toBackup(backup, id -> vm, this::getHostById, this::getBackupDisks); } catch (Exception e) { throw new CloudRuntimeException("Failed to finalize backup: " + e.getMessage(), e); } finally { @@ -1135,6 +1188,14 @@ public void finalizeBackup(final String vmUuid, final String uuid, String data) } } + protected List getBackupDisks(final BackupVO backup) { + List volumeInfos = backup.getBackedUpVolumes(); + if (CollectionUtils.isEmpty(volumeInfos)) { + return Collections.emptyList(); + } + return VolumeJoinVOToDiskConverter.toDiskListFromVolumeInfos(volumeInfos); + } + public List listCheckpointsByInstanceUuid(final String uuid) { throw new InvalidParameterValueException("Checkpoints for VM with ID " + uuid + " not implemented"); // UserVmVO vo = userVmDao.findByUuid(uuid); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index c13bacdfba0a..b69164d2d8d3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -123,7 +123,7 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); logger.info("Received POST request on /api/disks endpoint. Request-data: {}", data); // ToDo: remove try { Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 9c77a28e426f..bff16e00d822 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -113,8 +113,7 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); - logger.info("Received POST request on /api/imagetransfers endpoint. Request-data: {}", data); + String data = RouteHandler.getRequestData(req, logger); try { ImageTransfer request = io.getMapper().jsonMapper().readValue(data, ImageTransfer.class); ImageTransfer response = serverAdapter.handleCreateImageTransfer(request); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 9eb12fdf3968..4618aa2ae544 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -246,12 +246,6 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.notFound(resp, null, outFormat); } - protected String getRequestData(final HttpServletRequest req) { - String data = RouteHandler.getRequestData(req); - logger.info("Received method: {} request. Request-data: {}", req.getMethod(), data); - return data; - } - protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final VmListQuery q = fromRequest(req); @@ -310,7 +304,7 @@ protected static Integer parseIntOrNull(final String s) { protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); try { Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); Vm response = serverAdapter.createInstance(request); @@ -332,7 +326,7 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi protected void handleUpdateById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); logger.info("Received PUT request. Request-data: {}", data); try { Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); @@ -397,7 +391,7 @@ protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServlet protected void handlePostDiskAttachmentForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); try { DiskAttachment request = io.getMapper().jsonMapper().readValue(data, DiskAttachment.class); DiskAttachment response = serverAdapter.handleInstanceAttachDisk(id, request); @@ -421,7 +415,7 @@ protected void handleGetNicsByVmId(final String id, final HttpServletResponse re protected void handlePostNicForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); try { Nic request = io.getMapper().jsonMapper().readValue(data, Nic.class); Nic response = serverAdapter.handleAttachInstanceNic(id, request); @@ -445,7 +439,7 @@ protected void handleGetSnapshotsByVmId(final String id, final HttpServletRespon protected void handlePostSnapshotForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); try { Snapshot request = io.getMapper().jsonMapper().readValue(data, Snapshot.class); Snapshot response = serverAdapter.handleCreateInstanceSnapshot(id, request); @@ -486,7 +480,7 @@ protected void handleRestoreSnapshotById(final String id, final HttpServletReque final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { //ToDo: implement - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); io.badRequest(resp, "Not implemented", outFormat); } @@ -504,11 +498,11 @@ protected void handleGetBackupsByVmId(final String id, final HttpServletResponse protected void handlePostBackupForVmId(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = getRequestData(req); + String data = RouteHandler.getRequestData(req, logger); try { Backup request = io.getMapper().jsonMapper().readValue(data, Backup.class); Backup response = serverAdapter.createInstanceBackup(id, request); - io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } @@ -539,10 +533,9 @@ protected void handleGetBackupDisksById(final String id, final HttpServletReques protected void handleFinalizeBackupById(final String vmId, final String backupId, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = getRequestData(req); try { - serverAdapter.finalizeBackup(vmId, backupId, data); - io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); + Backup backup = serverAdapter.finalizeBackup(vmId, backupId); + io.getWriter().write(resp, HttpServletResponse.SC_OK, backup, outFormat); } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java index 5d93524ef528..728d38e6c31e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java @@ -25,13 +25,16 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Backup; +import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.Vm; +import com.cloud.api.query.vo.HostJoinVO; import com.cloud.vm.UserVmVO; public class BackupVOToBackupConverter { - public static Backup toBackup(final BackupVO backupVO, final Function vmResolver) { + public static Backup toBackup(final BackupVO backupVO, final Function vmResolver, + final Function hostResolver, final Function> disksResolver) { Backup backup = new Backup(); final String basePath = VeeamControlService.ContextPath.value(); backup.setHref(basePath + VmsRouteHandler.BASE_ROUTE + "/backups/" + backupVO.getUuid()); @@ -39,13 +42,13 @@ public static Backup toBackup(final BackupVO backupVO, final Function toBackupList(final List backupVOs, final Function vmResolver) { + public static List toBackupList(final List backupVOs, final Function vmResolver, + final Function hostResolver) { return backupVOs .stream() - .map(backupVO -> toBackup(backupVO, vmResolver)) + .map(backupVO -> toBackup(backupVO, vmResolver, hostResolver, null)) .collect(Collectors.toList()); } + + private static String mapStatusToPhase(final BackupVO.Status status) { + switch (status) { + case Allocated: + case Queued: + return "initializing"; + case BackingUp: + return "starting"; + case ReadyForTransfer: + return "ready"; + case FinalizingTransfer: + return "finalizing"; + case Restoring: + case BackedUp: + return "succeeded"; + } + return "failed"; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 1214ccd172af..2808e20a188c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -17,10 +17,12 @@ package org.apache.cloudstack.veeam.api.converter; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.DisksRouteHandler; @@ -133,6 +135,21 @@ public static List toDiskList(final List srcList) { .collect(Collectors.toList()); } + public static List toDiskListFromVolumeInfos(final List volumeInfos) { + List disks = new ArrayList<>(); + for (Backup.VolumeInfo volumeInfo : volumeInfos) { + Disk disk = new Disk(); + disk.setId(volumeInfo.getUuid()); + disk.setName(volumeInfo.getUuid()); + disk.setProvisionedSize(String.valueOf(volumeInfo.getSize())); + disk.setActualSize(String.valueOf(volumeInfo.getSize())); + disk.setTotalSize(String.valueOf(volumeInfo.getSize())); + disk.setBootable(String.valueOf(Volume.Type.ROOT.equals(volumeInfo.getType()))); + disks.add(disk); + } + return disks; + } + public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { final DiskAttachment da = new DiskAttachment(); final String basePath = VeeamControlService.ContextPath.value(); diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index b2e906aed4fc..40c459782b69 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -38,19 +38,19 @@ import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.CheckpointResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; @@ -131,7 +131,8 @@ private boolean isDummyOffering(Long backupOfferingId) { } @Override - public BackupResponse startBackup(StartBackupCmd cmd) { + public Backup createBackup(StartBackupCmd cmd) { + //ToDo: add config check, access check, resource count check, etc. Long vmId = cmd.getVmId(); VMInstanceVO vm = vmInstanceDao.findById(vmId); @@ -148,11 +149,17 @@ public BackupResponse startBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("Backup already in progress for VM: " + vmId); } - boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); - BackupVO backup = new BackupVO(); backup.setVmId(vmId); - backup.setName(vmId + "-" + DateTime.now()); + String name = cmd.getName(); + if (StringUtils.isEmpty(name)) { + name = vmId + "-" + DateTime.now(); + } + backup.setName(name); + final String description = cmd.getDescription(); + if (StringUtils.isNotEmpty(description)) { + backup.setDescription(description); + } backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); @@ -162,7 +169,6 @@ public BackupResponse startBackup(StartBackupCmd cmd) { String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); String fromCheckpointId = vm.getActiveCheckpointId(); - Long fromCheckpointCreateTime = vm.getActiveCheckpointCreateTime(); backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); @@ -174,27 +180,39 @@ public BackupResponse startBackup(StartBackupCmd cmd) { // Will be changed later if incremental was done backup.setType("FULL"); - backup = backupDao.persist(backup); + return backupDao.persist(backup); + } + @Override + public Backup startBackup(StartBackupCmd cmd) { + BackupVO backup = backupDao.findById(cmd.getEntityId()); + Long vmId = cmd.getVmId(); + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + throw new CloudRuntimeException("VM not found: " + vmId); + } List volumes = volumeDao.findByInstance(vmId); Map diskPathUuidMap = new HashMap<>(); for (Volume vol : volumes) { String volumePath = getVolumePathForFileBasedBackend(vol); diskPathUuidMap.put(volumePath, vol.getUuid()); } + long hostId = backup.getHostId(); Host host = hostDao.findById(hostId); StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), - toCheckpointId, - fromCheckpointId, - fromCheckpointCreateTime, - nbdPort, + backup.getToCheckpointId(), + backup.getFromCheckpointId(), + vm.getActiveCheckpointCreateTime(), + backup.getNbdPort(), diskPathUuidMap, host.getPrivateIpAddress(), vm.getState() == State.Stopped ); + boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); + StartBackupAnswer answer; try { if (dummyOffering) { @@ -218,13 +236,14 @@ public BackupResponse startBackup(StartBackupCmd cmd) { // todo: set it in the backend backup.setType("Incremental"); } + backup.setStatus(Backup.Status.ReadyForTransfer); backupDao.update(backup.getId(), backup); + return backup; + } - BackupResponse response = new BackupResponse(); - response.setId(backup.getUuid()); - response.setVmId(vm.getUuid()); - response.setStatus(backup.getStatus()); - return response; + protected void updateBackupState(BackupVO backup, Backup.Status newStatus) { + backup.setStatus(newStatus); + backupDao.update(backup.getId(), backup); } @Override @@ -249,9 +268,12 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); + updateBackupState(backup, Backup.Status.FinalizingTransfer); + List transfers = imageTransferDao.listByBackupId(backupId); for (ImageTransferVO transfer : transfers) { if (transfer.getPhase() != ImageTransferVO.Phase.finished) { + updateBackupState(backup, Backup.Status.Failed); throw new CloudRuntimeException(String.format("Image transfer %s not finalized for backup: %s", transfer.getUuid(), backup.getUuid())); } imageTransferDao.remove(transfer.getId()); @@ -269,10 +291,12 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } } catch (AgentUnavailableException | OperationTimedoutException e) { + updateBackupState(backup, Backup.Status.Failed); throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { + updateBackupState(backup, Backup.Status.Failed); throw new CloudRuntimeException("Failed to stop backup: " + answer.getDetails()); } } @@ -290,7 +314,8 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { } // Delete backup session record - backupDao.remove(backup.getId()); + backup.setStatus(Backup.Status.BackedUp); + backupDao.update(backupId, backup); return true; @@ -616,8 +641,7 @@ public boolean finalizeImageTransfer(final long imageTransferId) { } imageTransfer.setPhase(ImageTransferVO.Phase.finished); imageTransferDao.update(imageTransfer.getId(), imageTransfer); -// ToDo: check this -// imageTransferDao.remove(imageTransfer.getId()); + imageTransferDao.remove(imageTransfer.getId()); return true; } From 0b4b02da63a22dbfe0deb3de692413f76212e105 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Feb 2026 17:47:24 +0530 Subject: [PATCH 039/129] changes to backup and checkpoints api Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../admin/backup/DeleteVmCheckpointCmd.java | 10 +++- .../admin/backup/FinalizeBackupCmd.java | 46 +++++++++++++------ .../admin/backup/ListVmCheckpointsCmd.java | 2 +- .../api/response/CheckpointResponse.java | 21 +++++---- .../backup/IncrementalBackupService.java | 2 +- .../backup/IncrementalBackupServiceImpl.java | 25 ++++++---- 7 files changed, 71 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 2e686560a015..6ae349ca7128 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -332,6 +332,7 @@ public class ApiConstants { public static final String IS_2FA_VERIFIED = "is2faverified"; public static final String IS_2FA_MANDATED = "is2famandated"; + public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; public static final String IP_AVAILABLE = "ipavailable"; public static final String IP_LIMIT = "iplimit"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java index a05db27de4df..47b62ddcc501 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -30,7 +30,7 @@ import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.context.CallContext; -@APICommand(name = "deleteVmCheckpoint", +@APICommand(name = "deleteVirtualMachineCheckpoint", description = "Delete a VM checkpoint", responseObject = SuccessResponse.class, since = "4.22.0", @@ -61,6 +61,14 @@ public String getCheckpointId() { return checkpointId; } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setCheckpointId(String checkpointId) { + this.checkpointId = checkpointId; + } + @Override public void execute() { boolean result = incrementalBackupService.deleteVmCheckpoint(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java index 3ea69b66b5b0..e6e270c7f6f8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -22,25 +22,33 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.BackupResponse; -import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.IncrementalBackupService; import org.apache.cloudstack.context.CallContext; +import com.cloud.event.EventTypes; + @APICommand(name = "finalizeBackup", description = "Finalize a VM backup session", - responseObject = SuccessResponse.class, + responseObject = BackupResponse.class, since = "4.22.0", authorized = {RoleType.Admin}) -public class FinalizeBackupCmd extends BaseCmd implements AdminCmd { +public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { @Inject private IncrementalBackupService incrementalBackupService; + @Inject + private BackupManager backupManager; + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, @@ -63,19 +71,16 @@ public Long getBackupId() { return backupId; } - public void setVmId(Long vmId) { - this.vmId = vmId; - } - - public void setBackupId(Long backupId) { - this.backupId = backupId; - } - @Override public void execute() { - boolean result = incrementalBackupService.finalizeBackup(this); - SuccessResponse response = new SuccessResponse(getCommandName()); - response.setSuccess(result); + Backup backup = incrementalBackupService.finalizeBackup(this); + + if (backup == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + + BackupResponse response = backupManager.createBackupResponse(backup, null); + response.setResponseName(getCommandName()); setResponseObject(response); } @@ -84,4 +89,15 @@ public void execute() { public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Finalizing backup " + backupId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java index 737227bf6c7a..0d223ffaf5db 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -32,7 +32,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.IncrementalBackupService; -@APICommand(name = "listVmCheckpoints", +@APICommand(name = "listVirtualMachineCheckpoints", description = "List checkpoints for a VM", responseObject = CheckpointResponse.class, since = "4.22.0", diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java index 40be9d6d6d0a..2bec7711064f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.api.response; +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; @@ -24,24 +27,24 @@ public class CheckpointResponse extends BaseResponse { - @SerializedName("checkpointid") + @SerializedName(ApiConstants.ID) @Param(description = "the checkpoint ID") - private String checkpointId; + private String id; - @SerializedName("createtime") + @SerializedName(ApiConstants.CREATED) @Param(description = "the checkpoint creation time") - private Long createTime; + private Date created; - @SerializedName("isactive") + @SerializedName(ApiConstants.IS_ACTIVE) @Param(description = "whether this is the active checkpoint") private Boolean isActive; - public void setCheckpointId(String checkpointId) { - this.checkpointId = checkpointId; + public void setId(String id) { + this.id = id; } - public void setCreateTime(Long createTime) { - this.createTime = createTime; + public void setCreated(Date created) { + this.created = created; } public void setIsActive(Boolean isActive) { diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java index ed97f780db1c..053f1c1455e0 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java @@ -58,7 +58,7 @@ public interface IncrementalBackupService extends Configurable, PluggableService * Finalize a backup session * Stops NBD server, updates checkpoint tracking, deletes old checkpoints */ - boolean finalizeBackup(FinalizeBackupCmd cmd); + Backup finalizeBackup(FinalizeBackupCmd cmd); /** * Create an image transfer object for a disk diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 40c459782b69..be6dcae12b85 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.backup; +import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -247,7 +248,7 @@ protected void updateBackupState(BackupVO backup, Backup.Status newStatus) { } @Override - public boolean finalizeBackup(FinalizeBackupCmd cmd) { + public Backup finalizeBackup(FinalizeBackupCmd cmd) { Long vmId = cmd.getVmId(); Long backupId = cmd.getBackupId(); @@ -317,7 +318,7 @@ public boolean finalizeBackup(FinalizeBackupCmd cmd) { backup.setStatus(Backup.Status.BackedUp); backupDao.update(backupId, backup); - return true; + return backup; } @@ -673,14 +674,20 @@ public List listVmCheckpoints(ListVmCheckpointsCmd cmd) { // Return active checkpoint (POC: simplified, no libvirt query) List responses = new ArrayList<>(); - if (vm.getActiveCheckpointId() != null) { - CheckpointResponse response = new CheckpointResponse(); - response.setCheckpointId(vm.getActiveCheckpointId()); - response.setCreateTime(vm.getActiveCheckpointCreateTime()); - response.setIsActive(true); - responses.add(response); + if (vm.getActiveCheckpointId() == null) { + return responses; + } + CheckpointResponse response = new CheckpointResponse(); + response.setObjectName("checkpoint"); + response.setId(vm.getActiveCheckpointId()); + Long createTimeSeconds = vm.getActiveCheckpointCreateTime(); + if (createTimeSeconds != null) { + response.setCreated(Date.from(Instant.ofEpochSecond(createTimeSeconds))); + } else { + response.setCreated(new Date()); } - + response.setIsActive(true); + responses.add(response); return responses; } From c0b8aa636a8e837b606027449611ad95772828ae Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Feb 2026 17:48:03 +0530 Subject: [PATCH 040/129] plugin changes, fixes Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 91 +++++++++---------- .../cloudstack/veeam/api/VmsRouteHandler.java | 17 +--- .../converter/HostJoinVOToHostConverter.java | 2 +- .../api/converter/NicVOToNicConverter.java | 23 +++-- .../converter/UserVmJoinVOToVmConverter.java | 11 --- .../UserVmVOToCheckpointConverter.java | 45 +++++++++ .../apache/cloudstack/veeam/api/dto/Cpu.java | 6 +- .../veeam/api/dto/SummaryCount.java | 15 +-- .../cloudstack/veeam/api/dto/Topology.java | 24 ++--- 9 files changed, 133 insertions(+), 101 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 761abb3f0ab5..0d62758af670 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -37,6 +38,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; @@ -84,6 +86,7 @@ import org.apache.cloudstack.veeam.api.converter.NicVOToNicConverter; import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; +import org.apache.cloudstack.veeam.api.converter.UserVmVOToCheckpointConverter; import org.apache.cloudstack.veeam.api.converter.VmSnapshotVOToSnapshotConverter; import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; import org.apache.cloudstack.veeam.api.dto.Backup; @@ -442,7 +445,7 @@ public Vm createInstance(Vm request) { } Integer cpu = null; try { - cpu = request.getCpu().getTopology().getSockets(); + cpu = Integer.valueOf(request.getCpu().getTopology().getSockets()); } catch (Exception ignored) {} if (cpu == null) { throw new InvalidParameterValueException("CPU topology sockets must be specified"); @@ -1078,9 +1081,8 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - // Register a context as resource owner - Account account = accountService.getAccount(vmVo.getAccountId()); - CallContext ctx = CallContext.register(vmVo.getUserId(), vmVo.getAccountId()); + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartBackupCmd cmd = new StartBackupCmd(); ComponentContext.inject(cmd); @@ -1089,8 +1091,8 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { params.put(ApiConstants.NAME, request.getName()); params.put(ApiConstants.DESCRIPTION, request.getDescription()); ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, vmVo.getUserId(), account); - if (result.objectId == null) { + apiServerService.processAsyncCmd(cmd, params, ctx, vmVo.getUserId(), serviceUserAccount.second()); + if (result == null || result.objectId == null) { throw new CloudRuntimeException("Unexpected backup ID returned"); } BackupVO vo = backupDao.findById(result.objectId); @@ -1169,14 +1171,16 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { throw new InvalidParameterValueException("Backup with ID " + backupUuid + " not found"); } Pair serviceUserAccount = createServiceAccountIfNeeded(); - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { FinalizeBackupCmd cmd = new FinalizeBackupCmd(); ComponentContext.inject(cmd); - cmd.setBackupId(backup.getId()); - cmd.setVmId(vm.getId()); - boolean result = incrementalBackupService.finalizeBackup(cmd); - if (!result) { + Map params = new HashMap<>(); + params.put(ApiConstants.VIRTUAL_MACHINE_ID, vm.getUuid()); + params.put(ApiConstants.ID, backup.getUuid()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, vm.getUserId(), serviceUserAccount.second()); + if (result == null) { throw new CloudRuntimeException("Failed to finalize backup"); } backup = backupDao.findById(backup.getId()); @@ -1197,42 +1201,37 @@ protected List getBackupDisks(final BackupVO backup) { } public List listCheckpointsByInstanceUuid(final String uuid) { - throw new InvalidParameterValueException("Checkpoints for VM with ID " + uuid + " not implemented"); -// UserVmVO vo = userVmDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); -// } -// List checkpoints = checkpointDao.findByVmId(vo.getId()); -// return CheckpointVOToCheckpointConverter.toCheckpointList(checkpoints, vo.getUuid()); + UserVmVO vo = userVmDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + Checkpoint checkpoint = UserVmVOToCheckpointConverter.toCheckpoint(vo); + if (checkpoint == null) { + return Collections.emptyList(); + } + return List.of(checkpoint); } - public ResourceAction deleteCheckpoint(String uuid, boolean async) { - throw new InvalidParameterValueException("Delete Checkpoint with ID " + uuid + " not implemented"); -// ResourceAction action = null; -// CheckpointVO vo = checkpointDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Checkpoint with ID " + uuid + " not found"); -// } -// Pair serviceUserAccount = createServiceAccountIfNeeded(); -// CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); -// try { -// DeleteCheckpointCmd cmd = new DeleteCheckpointCmd(); -// ComponentContext.inject(cmd); -// Map params = new HashMap<>(); -// params.put(ApiConstants.CHECKPOINT_ID, vo.getUuid()); -// ApiServerService.AsyncCmdResult result = -// apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), -// serviceUserAccount.second()); -// AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); -// if (jobVo == null) { -// throw new CloudRuntimeException("Failed to find job for checkpoint deletion"); -// } -// action = AsyncJobJoinVOToJobConverter.toAction(jobVo); -// } catch (Exception e) { -// throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); -// } finally { -// CallContext.unregister(); -// } -// return action; + public void deleteCheckpoint(String vmUuid, String checkpointId) { + UserVmVO vo = userVmDao.findByUuid(vmUuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); + } + if (!Objects.equals(vo.getActiveCheckpointId(), checkpointId)) { + logger.warn("Checkpoint ID {} does not match active checkpoint for VM {}", checkpointId, vmUuid); + return; + } + Pair serviceUserAccount = createServiceAccountIfNeeded(); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + DeleteVmCheckpointCmd cmd = new DeleteVmCheckpointCmd(); + ComponentContext.inject(cmd); + cmd.setVmId(vo.getId()); + incrementalBackupService.deleteVmCheckpoint(cmd); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 4618aa2ae544..1b66b37e4318 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -30,7 +30,6 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Backup; import org.apache.cloudstack.veeam.api.dto.Checkpoint; -import org.apache.cloudstack.veeam.api.dto.Checkpoints; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.DiskAttachments; @@ -208,7 +207,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path return; } else if ("checkpoints".equals(subPath)) { if ("DELETE".equalsIgnoreCase(method)) { - handleDeleteCheckpointById(subId, req, resp, outFormat, io); + handleDeleteCheckpoint(id, subId, resp, outFormat, io); } else { io.methodNotAllowed(resp, "DELETE", outFormat); } @@ -545,25 +544,19 @@ protected void handleGetCheckpointsByVmId(final String id, final HttpServletResp final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { List checkpoints = serverAdapter.listCheckpointsByInstanceUuid(id); - Checkpoints response = new Checkpoints(checkpoints); + NamedList response = NamedList.of("checkpoints", checkpoints); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); } } - protected void handleDeleteCheckpointById(final String id, final HttpServletRequest req, + protected void handleDeleteCheckpoint(final String vmId, final String checkpointId, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String asyncStr = req.getParameter("async"); - boolean async = !Boolean.FALSE.toString().equals(asyncStr); try { - ResourceAction action = serverAdapter.deleteCheckpoint(id, async); - if (action != null) { - io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, action, outFormat); - } else { - io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); - } + serverAdapter.deleteCheckpoint(vmId, checkpointId); + io.getWriter().write(resp, HttpServletResponse.SC_OK, null, outFormat); } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java index 6f4acbd4550b..d627aa4d63ff 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java @@ -66,7 +66,7 @@ public static Host toHost(final HostJoinVO vo) { // --- CPU --- final Cpu cpu = new Cpu(); - cpu.setSpeed(Math.toIntExact(vo.getSpeed())); + cpu.setSpeed(String.valueOf(Math.toIntExact(vo.getSpeed()))); final Topology topo = new Topology(vo.getCpuSockets(), vo.getCpus(), 1); cpu.setTopology(topo); h.setCpu(cpu); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java index 1eb5eaf29cb5..7ccaf45e2fdf 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.veeam.api.dto.ReportedDevice; import org.apache.cloudstack.veeam.api.dto.ReportedDevices; import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; @@ -76,17 +77,19 @@ private static ReportedDevice getReportedDevice(NicVO vo, Mac mac, Vm vm) { device.setName("eth0"); device.setDescription(String.format("%s device", vo.getReserver())); device.setMac(mac); - Ip ip = new Ip(); - if (vo.getIPv4Address() != null) { - ip.setAddress(vo.getIPv4Address()); - ip.setGateway(vo.getIPv4Gateway()); - ip.setVersion("v4"); - } else if (vo.getIPv6Address() != null) { - ip.setAddress(vo.getIPv6Address()); - ip.setGateway(vo.getIPv6Gateway()); - ip.setVersion("v6"); + if (ObjectUtils.anyNotNull(vo.getIPv4Address(), vo.getIPv6Address())) { + Ip ip = new Ip(); + if (vo.getIPv4Address() != null) { + ip.setAddress(vo.getIPv4Address()); + ip.setGateway(vo.getIPv4Gateway()); + ip.setVersion("v4"); + } else if (vo.getIPv6Address() != null) { + ip.setAddress(vo.getIPv6Address()); + ip.setGateway(vo.getIPv6Gateway()); + ip.setVersion("v6"); + } + device.setIps(new Ips(List.of(ip))); } - device.setIps(new Ips(List.of(ip))); device.setHref(vm.getHref() + "/reporteddevices/" + vo.getUuid()); device.setVm(vm); return device; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 09e058a3eaa3..03a7ead0cc13 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -25,7 +25,6 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; -import org.apache.cloudstack.veeam.api.JobsRouteHandler; import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.BaseDto; @@ -40,7 +39,6 @@ import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; -import org.apache.cloudstack.veeam.api.dto.VmAction; import org.apache.commons.lang3.StringUtils; import com.cloud.api.query.vo.HostJoinVO; @@ -156,15 +154,6 @@ public static List toVmList(final List srcList, final Function .collect(Collectors.toList()); } - public static VmAction toVmAction(final UserVmJoinVO vm) { - VmAction action = new VmAction(); - final String basePath = VeeamControlService.ContextPath.value(); - action.setVm(toVm(vm, null, null, null)); - action.setJob(Ref.of(basePath + JobsRouteHandler.BASE_ROUTE + vm.getUuid(), vm.getUuid())); - action.setStatus("complete"); - return action; - } - private static String mapStatus(final VirtualMachine.State state) { // CloudStack-ish states -> oVirt-ish up/down if (Arrays.asList(VirtualMachine.State.Running, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java new file mode 100644 index 000000000000..019bc8264c84 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.time.Instant; + +import org.apache.cloudstack.veeam.api.dto.Checkpoint; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.vm.UserVmVO; + +public class UserVmVOToCheckpointConverter { + + public static Checkpoint toCheckpoint(final UserVmVO vm) { + if (StringUtils.isEmpty(vm.getActiveCheckpointId())) { + return null; + } + Checkpoint checkpoint = new Checkpoint(); + checkpoint.setId(vm.getActiveCheckpointId()); + checkpoint.setName(vm.getActiveCheckpointId()); + Long createTimeSeconds = vm.getActiveCheckpointCreateTime(); + if (createTimeSeconds != null) { + checkpoint.setCreationDate(String.valueOf(Instant.ofEpochSecond(createTimeSeconds).toEpochMilli())); + } else { + checkpoint.setCreationDate(String.valueOf(System.currentTimeMillis())); + } + checkpoint.setState("created"); + return checkpoint; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java index 97459b40cd8d..c5cea76f33e9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java @@ -22,15 +22,15 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Cpu { private String name; - private Integer speed; + private String speed; private String architecture; private String type; private Topology topology; public String getName() { return name; } public void setName(String name) { this.name = name; } - public Integer getSpeed() { return speed; } - public void setSpeed(Integer speed) { this.speed = speed; } + public String getSpeed() { return speed; } + public void setSpeed(String speed) { this.speed = speed; } public String getArchitecture() { return architecture; } public void setArchitecture(String architecture) { this.architecture = architecture; } public String getType() { return type; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java index 280704f9b51c..ac26619ff026 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java @@ -22,19 +22,22 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class SummaryCount { - private Integer active; - private Integer total; + private String active; + private String total; + + public SummaryCount() { + } public SummaryCount(Integer active, Integer total) { - this.active = active; - this.total = total; + this.active = String.valueOf(active); + this.total = String.valueOf(total); } - public Integer getActive() { + public String getActive() { return active; } - public Integer getTotal() { + public String getTotal() { return total; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java index 564df5b53048..fa20db9d658c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java @@ -21,40 +21,40 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Topology { - public Integer sockets; - public Integer cores; - public Integer threads; + public String sockets; + public String cores; + public String threads; public Topology() { } public Topology(final Integer sockets, final Integer cores, final Integer threads) { - this.sockets = sockets; - this.cores = cores; - this.threads = threads; + this.sockets = String.valueOf(sockets); + this.cores = String.valueOf(cores); + this.threads = String.valueOf(threads); } - public Integer getSockets() { + public String getSockets() { return sockets; } - public void setSockets(Integer sockets) { + public void setSockets(String sockets) { this.sockets = sockets; } - public Integer getCores() { + public String getCores() { return cores; } - public void setCores(Integer cores) { + public void setCores(String cores) { this.cores = cores; } - public Integer getThreads() { + public String getThreads() { return threads; } - public void setThreads(Integer threads) { + public void setThreads(String threads) { this.threads = threads; } } From 3a02433d75f9152480c7183effeb7da70b0b5ee9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 18 Feb 2026 09:00:04 +0530 Subject: [PATCH 041/129] refactor Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 8 +- .../veeam/api/ClustersRouteHandler.java | 5 +- .../veeam/api/DataCentersRouteHandler.java | 12 +- .../veeam/api/DisksRouteHandler.java | 5 +- .../veeam/api/HostsRouteHandler.java | 5 +- .../veeam/api/ImageTransfersRouteHandler.java | 5 +- .../veeam/api/JobsRouteHandler.java | 5 +- .../veeam/api/NetworksRouteHandler.java | 5 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 15 +- .../veeam/api/VnicProfilesRouteHandler.java | 5 +- .../AsyncJobJoinVOToJobConverter.java | 3 - .../ClusterVOToClusterConverter.java | 6 - ...ageTransferVOToImageTransferConverter.java | 4 +- .../NetworkVOToNetworkConverter.java | 4 +- .../api/converter/NicVOToNicConverter.java | 7 +- .../StoreVOToStorageDomainConverter.java | 6 +- .../converter/UserVmJoinVOToVmConverter.java | 7 +- .../VmSnapshotVOToSnapshotConverter.java | 4 +- .../VolumeJoinVOToDiskConverter.java | 15 +- .../cloudstack/veeam/api/dto/Actions.java | 41 --- .../cloudstack/veeam/api/dto/Backups.java | 32 --- .../cloudstack/veeam/api/dto/Certificate.java | 19 +- .../cloudstack/veeam/api/dto/Checkpoints.java | 42 --- .../cloudstack/veeam/api/dto/Cluster.java | 6 +- .../cloudstack/veeam/api/dto/Clusters.java | 46 --- .../apache/cloudstack/veeam/api/dto/Cpu.java | 49 +++- .../cloudstack/veeam/api/dto/DataCenter.java | 6 +- .../cloudstack/veeam/api/dto/DataCenters.java | 49 ---- .../apache/cloudstack/veeam/api/dto/Disk.java | 12 +- .../veeam/api/dto/DiskAttachment.java | 3 +- .../veeam/api/dto/DiskAttachments.java | 38 --- .../cloudstack/veeam/api/dto/Disks.java | 38 --- .../veeam/api/dto/EmptyElement.java | 3 +- .../veeam/api/dto/HardwareInformation.java | 49 +++- .../apache/cloudstack/veeam/api/dto/Host.java | 261 ++++++++++++++---- .../cloudstack/veeam/api/dto/HostSummary.java | 29 +- .../cloudstack/veeam/api/dto/Hosts.java | 33 --- .../veeam/api/dto/ImageTransfer.java | 6 +- .../veeam/api/dto/ImageTransfers.java | 39 --- .../apache/cloudstack/veeam/api/dto/Ips.java | 42 --- .../apache/cloudstack/veeam/api/dto/Job.java | 92 ++++-- .../apache/cloudstack/veeam/api/dto/Jobs.java | 42 --- .../cloudstack/veeam/api/dto/NamedList.java | 6 + .../cloudstack/veeam/api/dto/Network.java | 95 +++++-- .../veeam/api/dto/NetworkUsages.java | 42 --- .../cloudstack/veeam/api/dto/Networks.java | 33 --- .../apache/cloudstack/veeam/api/dto/Nic.java | 6 +- .../apache/cloudstack/veeam/api/dto/Nics.java | 3 +- .../veeam/api/dto/ReportedDevice.java | 6 +- .../veeam/api/dto/ReportedDevices.java | 42 --- .../cloudstack/veeam/api/dto/Snapshot.java | 9 +- .../cloudstack/veeam/api/dto/Snapshots.java | 41 --- .../veeam/api/dto/StorageDomain.java | 12 +- .../veeam/api/dto/StorageDomains.java | 42 --- .../cloudstack/veeam/api/dto/Version.java | 3 +- .../apache/cloudstack/veeam/api/dto/Vm.java | 12 +- .../apache/cloudstack/veeam/api/dto/Vms.java | 45 --- .../veeam/api/dto/VnicProfiles.java | 49 ---- 58 files changed, 565 insertions(+), 984 deletions(-) delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 0d62758af670..781cb6b94d66 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -533,6 +533,7 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, l } public Vm updateInstance(String uuid, Vm request) { + // ToDo: what to do?! return getInstance(uuid); } @@ -724,11 +725,11 @@ public Disk handleCreateDisk(Disk request) { if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { throw new InvalidParameterValueException("Only worker VM disk creation is supported"); } - if (request.getStorageDomains() == null || CollectionUtils.isEmpty(request.getStorageDomains().getStorageDomain()) || - request.getStorageDomains().getStorageDomain().size() > 1) { + if (request.getStorageDomains() == null || CollectionUtils.isEmpty(request.getStorageDomains().getItems()) || + request.getStorageDomains().getItems().size() > 1) { throw new InvalidParameterValueException("Exactly one storage domain must be specified"); } - StorageDomain domain = request.getStorageDomains().getStorageDomain().get(0); + StorageDomain domain = request.getStorageDomains().getItems().get(0); if (domain == null || domain.getId() == null) { throw new InvalidParameterValueException("Storage domain ID must be specified"); } @@ -1154,6 +1155,7 @@ public Backup getBackup(String uuid) { public List listDisksByBackupUuid(final String uuid) { throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implemented"); +// ToDo: implement // BackupVO vo = backupDao.findByUuid(uuid); // if (vo == null) { // throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index 37ef228db9f3..c3ee3ab3cdd7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Cluster; -import org.apache.cloudstack.veeam.api.dto.Clusters; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,8 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllClusters(); - final Clusters response = new Clusters(result); - + NamedList response = NamedList.of("cluster", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 1b9e2e014014..bf8e2885251b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -28,11 +28,9 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.DataCenter; -import org.apache.cloudstack.veeam.api.dto.DataCenters; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Network; -import org.apache.cloudstack.veeam.api.dto.Networks; import org.apache.cloudstack.veeam.api.dto.StorageDomain; -import org.apache.cloudstack.veeam.api.dto.StorageDomains; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -99,8 +97,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllDataCenters(); - final DataCenters response = new DataCenters(result); - + NamedList response = NamedList.of("data_center", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } @@ -118,8 +115,7 @@ protected void handleGetStorageDomainsByDcId(final String id, final HttpServletR final VeeamControlServlet io) throws IOException { try { List storageDomains = serverAdapter.listStorageDomainsByDcId(id); - StorageDomains response = new StorageDomains(); - response.setStorageDomain(storageDomains); + NamedList response = NamedList.of("storage_domain", storageDomains); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -130,7 +126,7 @@ protected void handleGetNetworksByDcId(final String id, final HttpServletRespons final VeeamControlServlet io) throws IOException { try { List networks = serverAdapter.listNetworksByDcId(id); - Networks response = new Networks(networks); + NamedList response = NamedList.of("network", networks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index b69164d2d8d3..011dfe9d1b06 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Disk; -import org.apache.cloudstack.veeam.api.dto.Disks; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -116,8 +116,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllDisks(); - final Disks response = new Disks(result); - + NamedList response = NamedList.of("disk", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java index efe41bfbe301..54f19424cf93 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Host; -import org.apache.cloudstack.veeam.api.dto.Hosts; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,8 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllHosts(); - final Hosts response = new Hosts(result); - + NamedList response = NamedList.of("host", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index bff16e00d822..6a26d54beaf7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.ImageTransfer; -import org.apache.cloudstack.veeam.api.dto.ImageTransfers; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -106,8 +106,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllImageTransfers(); - final ImageTransfers response = new ImageTransfers(); - response.setImageTransfer(result); + NamedList response = NamedList.of("image_transfer", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index 7213cdac5bed..a96c80aefe5b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Job; -import org.apache.cloudstack.veeam.api.dto.Jobs; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,8 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllJobs(); - final Jobs response = new Jobs(result); - + NamedList response = NamedList.of("job", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index 2450c85cf517..5e5d9927e65a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -27,8 +27,8 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Network; -import org.apache.cloudstack.veeam.api.dto.Networks; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,8 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllNetworks(); - final Networks response = new Networks(result); - + NamedList response = NamedList.of("network", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 1b66b37e4318..70a34ba08a64 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -32,17 +32,13 @@ import org.apache.cloudstack.veeam.api.dto.Checkpoint; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; -import org.apache.cloudstack.veeam.api.dto.DiskAttachments; -import org.apache.cloudstack.veeam.api.dto.Disks; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; -import org.apache.cloudstack.veeam.api.dto.Snapshots; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.dto.VmAction; -import org.apache.cloudstack.veeam.api.dto.Vms; import org.apache.cloudstack.veeam.api.request.VmListQuery; import org.apache.cloudstack.veeam.api.request.VmSearchExpr; import org.apache.cloudstack.veeam.api.request.VmSearchFilters; @@ -279,8 +275,7 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse } final List result = serverAdapter.listAllInstances(); - final Vms response = new Vms(result); - + NamedList response = NamedList.of("vm", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } @@ -380,7 +375,7 @@ protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServlet final VeeamControlServlet io) throws IOException { try { List disks = serverAdapter.listDiskAttachmentsByInstanceUuid(id); - DiskAttachments response = new DiskAttachments(disks); + NamedList response = NamedList.of("disk_attachment", disks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -428,7 +423,7 @@ protected void handleGetSnapshotsByVmId(final String id, final HttpServletRespon final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { List snapshots = serverAdapter.listSnapshotsByInstanceUuid(id); - Snapshots response = new Snapshots(snapshots); + NamedList response = NamedList.of("snapshot", snapshots); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -487,7 +482,7 @@ protected void handleGetBackupsByVmId(final String id, final HttpServletResponse final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { List backups = serverAdapter.listBackupsByInstanceUuid(id); - NamedList response = NamedList.of("backups", backups); + NamedList response = NamedList.of("backup", backups); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -522,7 +517,7 @@ protected void handleGetBackupDisksById(final String id, final HttpServletReques throws IOException { try { List disks = serverAdapter.listDisksByBackupUuid(id); - Disks response = new Disks(disks); + NamedList response = NamedList.of("disk", disks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index a0ce779d6446..28f6b816d14b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -27,8 +27,8 @@ import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.adapter.ServerAdapter; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.VnicProfile; -import org.apache.cloudstack.veeam.api.dto.VnicProfiles; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,8 +85,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final List result = serverAdapter.listAllVnicProfiles(); - final VnicProfiles response = new VnicProfiles(result); - + NamedList response = NamedList.of("vnic_profile", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index c66e9f78d0f7..bdae4983694e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -22,7 +22,6 @@ import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.JobsRouteHandler; -import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Job; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.ResourceAction; @@ -48,7 +47,6 @@ public static Job toJob(String uuid, String state, long startTime) { job.setEndTime(System.currentTimeMillis()); } job.setOwner(Ref.of(basePath + "/api/users/" + uuid, uuid)); - job.setActions(new Actions()); job.setDescription("Something"); job.setLink(Collections.emptyList()); return job; @@ -80,7 +78,6 @@ public static Job toJob(AsyncJobJoinVO vo) { job.setEndTime(endTime); } job.setOwner(Ref.of(basePath + "/api/users/" + vo.getUserUuid(), vo.getUserUuid())); - job.setActions(new Actions()); job.setDescription("Something"); job.setLink(Collections.emptyList()); return job; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java index c6a43068562e..7b532f26c02e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -25,7 +25,6 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ClustersRouteHandler; import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; -import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.Cpu; import org.apache.cloudstack.veeam.api.dto.Link; @@ -143,11 +142,6 @@ public static Cluster toCluster(final ClusterVO vo, final Function links = new ArrayList<>(); links.add(getLink(imageTransfer, "cancel")); links.add(getLink(imageTransfer, "finalize")); - imageTransfer.setActions(new Actions(links)); + imageTransfer.setActions(NamedList.of("link", links)); return imageTransfer; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java index 85775b3d6cfe..114311225d33 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java @@ -25,8 +25,8 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.DataCentersRouteHandler; import org.apache.cloudstack.veeam.api.NetworksRouteHandler; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Network; -import org.apache.cloudstack.veeam.api.dto.NetworkUsages; import org.apache.cloudstack.veeam.api.dto.Ref; import com.cloud.api.query.vo.DataCenterJoinVO; @@ -50,7 +50,7 @@ public static Network toNetwork(final NetworkVO vo, final Function h if (disksResolver != null) { List diskAttachments = disksResolver.apply(src.getId()); - dst.setDiskAttachments(new DiskAttachments(diskAttachments)); + dst.setDiskAttachments(NamedList.of("disk_attachment", diskAttachments)); } if (disksResolver != null) { @@ -132,7 +131,7 @@ public static Vm toVm(final UserVmJoinVO src, final Function h dst.setNics(new Nics(nics)); } - dst.setActions(new Actions(List.of( + dst.setActions(NamedList.of("link", List.of( BaseDto.getActionLink("start", dst.getHref()), BaseDto.getActionLink("stop", dst.getHref()), BaseDto.getActionLink("shutdown", dst.getHref()) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java index 7d1727d742a0..4dbc71505d79 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java @@ -22,8 +22,8 @@ import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.VmsRouteHandler; -import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.BaseDto; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -42,7 +42,7 @@ public static Snapshot toSnapshot(final VMSnapshotVO vmSnapshotVO, String vmUuid snapshot.setDate(vmSnapshotVO.getCreated().getTime()); snapshot.setPersistMemorystate(String.valueOf(VMSnapshotVO.Type.DiskAndMemory.equals(vmSnapshotVO.getType()))); snapshot.setSnapshotStatus(VMSnapshot.State.Ready.equals(vmSnapshotVO.getState()) ? "ok" : "locked"); - snapshot.setActions(new Actions(List.of(BaseDto.getActionLink("restore", snapshot.getHref())))); + snapshot.setActions(NamedList.of("link", List.of(BaseDto.getActionLink("restore", snapshot.getHref())))); return snapshot; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 2808e20a188c..497f4d7f441b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.veeam.api.converter; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -27,13 +26,12 @@ import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.DisksRouteHandler; import org.apache.cloudstack.veeam.api.VmsRouteHandler; -import org.apache.cloudstack.veeam.api.dto.Actions; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.Link; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.StorageDomain; -import org.apache.cloudstack.veeam.api.dto.StorageDomains; import org.apache.cloudstack.veeam.api.dto.Vm; import com.cloud.api.ApiDBUtils; @@ -110,17 +108,12 @@ public static Disk toDisk(final VolumeJoinVO vol) { // Storage domains if (vol.getPoolUuid() != null) { - StorageDomains sds = new StorageDomains(); StorageDomain sd = new StorageDomain(); sd.setHref(apiBasePath + "/storagedomains/" + vol.getPoolUuid()); sd.setId(vol.getPoolUuid()); - sds.setStorageDomain(List.of(sd)); - disk.setStorageDomains(sds); + disk.setStorageDomains(NamedList.of("storage_domain", List.of(sd))); } - // Actions (Veeam checks presence, not behavior) - disk.setActions(defaultDiskActions(diskHref)); - // Links disk.setLink(List.of( Link.of("disksnapshots", diskHref + "/disksnapshots") @@ -205,8 +198,4 @@ private static String mapStatus(final Volume.State state) { return "locked"; } } - - private static Actions defaultDiskActions(final String diskHref) { - return new Actions(Collections.emptyList()); - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java deleted file mode 100644 index 05767e5219d7..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Actions.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class Actions { - private List link; - - public Actions() {} - - public Actions(final List link) { - this.link = link; - } - - public List getLink() { - return link; - } - - public void setLink(List link) { - this.link = link; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java deleted file mode 100644 index c1cb39ef5f23..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backups.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -public class Backups { - - @JacksonXmlElementWrapper(useWrapping = false) - public List backup; - - public Backups(final List backup) { - this.backup = backup; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java index 7a87bfb09492..12e99159bfc9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java @@ -24,8 +24,19 @@ public class Certificate { private String organization; private String subject; - public String getOrganization() { return organization; } - public void setOrganization(String organization) { this.organization = organization; } - public String getSubject() { return subject; } - public void setSubject(String subject) { this.subject = subject; } + public String getOrganization() { + return organization; + } + + public void setOrganization(String organization) { + this.organization = organization; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java deleted file mode 100644 index 7cc346202a91..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoints.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -public class Checkpoints { - - @JacksonXmlElementWrapper(useWrapping = false) - private List checkpoint; - - public Checkpoints() {} - - public Checkpoints(final List checkpoint) { - this.checkpoint = checkpoint; - } - - public List getCheckpoint() { - return checkpoint; - } - - public void setCheckpoint(List checkpoint) { - this.checkpoint = checkpoint; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java index 650177a5e45b..db0cd8be6eab 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java @@ -58,7 +58,7 @@ public final class Cluster extends BaseDto { private Ref dataCenter; private Ref macPool; private Ref schedulingPolicy; - private Actions actions; + private NamedList actions; @JacksonXmlElementWrapper(useWrapping = false) private List link; @@ -310,11 +310,11 @@ public void setSchedulingPolicy(Ref schedulingPolicy) { this.schedulingPolicy = schedulingPolicy; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java deleted file mode 100644 index 4755962bd01d..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Clusters.java +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class Clusters { - - @JsonProperty("cluster") - @JacksonXmlElementWrapper(useWrapping = false) - private List cluster; - - public Clusters() {} - - public Clusters(final List cluster) { - this.cluster = cluster; - } - - public List getCluster() { - return cluster; - } - - public void setCluster(List cluster) { - this.cluster = cluster; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java index c5cea76f33e9..3dce4931c848 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java @@ -27,14 +27,43 @@ public final class Cpu { private String type; private Topology topology; - public String getName() { return name; } - public void setName(String name) { this.name = name; } - public String getSpeed() { return speed; } - public void setSpeed(String speed) { this.speed = speed; } - public String getArchitecture() { return architecture; } - public void setArchitecture(String architecture) { this.architecture = architecture; } - public String getType() { return type; } - public void setType(String type) { this.type = type; } - public Topology getTopology() { return topology; } - public void setTopology(Topology topology) { this.topology = topology; } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSpeed() { + return speed; + } + + public void setSpeed(String speed) { + this.speed = speed; + } + + public String getArchitecture() { + return architecture; + } + + public void setArchitecture(String architecture) { + this.architecture = architecture; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Topology getTopology() { + return topology; + } + + public void setTopology(Topology topology) { + this.topology = topology; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java index 9c3aed49406d..52f6a6c279f1 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java @@ -33,7 +33,7 @@ public final class DataCenter extends BaseDto { private SupportedVersions supportedVersions; private Version version; private Ref macPool; - private Actions actions; + private NamedList actions; private String name; private String description; @JacksonXmlElementWrapper(useWrapping = false) @@ -95,11 +95,11 @@ public void setMacPool(Ref macPool) { this.macPool = macPool; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java deleted file mode 100644 index fa44bbf86fc7..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenters.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -/** - * Root collection wrapper: - * { - * "data_center": [ { ... } ] - * } - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class DataCenters { - - @JacksonXmlElementWrapper(useWrapping = false) - public List dataCenter; - - public DataCenters() {} - public DataCenters(final List dataCenter) { - this.dataCenter = dataCenter; - } - - public List getDataCenter() { - return dataCenter; - } - - public void setDataCenter(List dataCenter) { - this.dataCenter = dataCenter; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java index ce609592f154..c9a19794c189 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java @@ -46,8 +46,8 @@ public final class Disk extends BaseDto { private String wipeAfterDelete; private Ref diskProfile; private Ref quota; - private StorageDomains storageDomains; - private Actions actions; + private NamedList storageDomains; + private NamedList actions; private String name; private String description; @JacksonXmlElementWrapper(useWrapping = false) @@ -205,19 +205,19 @@ public void setQuota(Ref quota) { this.quota = quota; } - public StorageDomains getStorageDomains() { + public NamedList getStorageDomains() { return storageDomains; } - public void setStorageDomains(StorageDomains storageDomains) { + public void setStorageDomains(NamedList storageDomains) { this.storageDomains = storageDomains; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java index 5b0428efb1b0..f22168342e35 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java @@ -34,7 +34,8 @@ public final class DiskAttachment extends BaseDto { private Disk disk; private Vm vm; - public DiskAttachment() {} + public DiskAttachment() { + } public String getActive() { return active; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java deleted file mode 100644 index 827a277ee70a..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachments.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class DiskAttachments { - - @JacksonXmlElementWrapper(useWrapping = false) - private List diskAttachment; - - public DiskAttachments(final List diskAttachment) { - this.diskAttachment = diskAttachment; - } - - public List getDiskAttachment() { - return diskAttachment; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java deleted file mode 100644 index a033d88899a0..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disks.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class Disks { - - @JacksonXmlElementWrapper(useWrapping = false) - private List disk; - - public Disks(final List disk) { - this.disk = disk; - } - - public List getDisk() { - return disk; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java index 54d65d8529bc..3c4111c55a31 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java @@ -21,5 +21,6 @@ @JsonSerialize(using = EmptyElementSerializer.class) public final class EmptyElement { - public EmptyElement() {} + public EmptyElement() { + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java index acddcfd30b15..0ded2f095f35 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java @@ -27,14 +27,43 @@ public class HardwareInformation { private String uuid; private String version; - public String getManufacturer() { return manufacturer; } - public void setManufacturer(String manufacturer) { this.manufacturer = manufacturer; } - public String getProductName() { return productName; } - public void setProductName(String productName) { this.productName = productName; } - public String getSerialNumber() { return serialNumber; } - public void setSerialNumber(String serialNumber) { this.serialNumber = serialNumber; } - public String getUuid() { return uuid; } - public void setUuid(String uuid) { this.uuid = uuid; } - public String getVersion() { return version; } - public void setVersion(String version) { this.version = version; } + public String getManufacturer() { + return manufacturer; + } + + public void setManufacturer(String manufacturer) { + this.manufacturer = manufacturer; + } + + public String getProductName() { + return productName; + } + + public void setProductName(String productName) { + this.productName = productName; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java index 5e37b7bf935b..c937cdb564ba 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java @@ -46,62 +46,217 @@ public class Host extends BaseDto { private Version version; private String vgpuPlacement; private Ref cluster; - private Actions actions; + private NamedList actions; private String name; private String comment; private List link; // getters/setters (generate via IDE) - public String getAddress() { return address; } - public void setAddress(String address) { this.address = address; } - public String getAutoNumaStatus() { return autoNumaStatus; } - public void setAutoNumaStatus(String autoNumaStatus) { this.autoNumaStatus = autoNumaStatus; } - public Certificate getCertificate() { return certificate; } - public void setCertificate(Certificate certificate) { this.certificate = certificate; } - public Cpu getCpu() { return cpu; } - public void setCpu(Cpu cpu) { this.cpu = cpu; } - public String getExternalStatus() { return externalStatus; } - public void setExternalStatus(String externalStatus) { this.externalStatus = externalStatus; } - public HardwareInformation getHardwareInformation() { return hardwareInformation; } - public void setHardwareInformation(HardwareInformation hardwareInformation) { this.hardwareInformation = hardwareInformation; } - public String getKdumpStatus() { return kdumpStatus; } - public void setKdumpStatus(String kdumpStatus) { this.kdumpStatus = kdumpStatus; } - public Version getLibvirtVersion() { return libvirtVersion; } - public void setLibvirtVersion(Version libvirtVersion) { this.libvirtVersion = libvirtVersion; } - public String getMaxSchedulingMemory() { return maxSchedulingMemory; } - public void setMaxSchedulingMemory(String maxSchedulingMemory) { this.maxSchedulingMemory = maxSchedulingMemory; } - public String getMemory() { return memory; } - public void setMemory(String memory) { this.memory = memory; } - public String getNumaSupported() { return numaSupported; } - public void setNumaSupported(String numaSupported) { this.numaSupported = numaSupported; } - public Os getOs() { return os; } - public void setOs(Os os) { this.os = os; } - public String getPort() { return port; } - public void setPort(String port) { this.port = port; } - public String getProtocol() { return protocol; } - public void setProtocol(String protocol) { this.protocol = protocol; } - public String getReinstallationRequired() { return reinstallationRequired; } - public void setReinstallationRequired(String reinstallationRequired) { this.reinstallationRequired = reinstallationRequired; } - public String getStatus() { return status; } - public void setStatus(String status) { this.status = status; } - public ApiSummary getSummary() { return summary; } - public void setSummary(ApiSummary summary) { this.summary = summary; } - public String getType() { return type; } - public void setType(String type) { this.type = type; } - public String getUpdateAvailable() { return updateAvailable; } - public void setUpdateAvailable(String updateAvailable) { this.updateAvailable = updateAvailable; } - public Version getVersion() { return version; } - public void setVersion(Version version) { this.version = version; } - public String getVgpuPlacement() { return vgpuPlacement; } - public void setVgpuPlacement(String vgpuPlacement) { this.vgpuPlacement = vgpuPlacement; } - public Ref getCluster() { return cluster; } - public void setCluster(Ref cluster) { this.cluster = cluster; } - public Actions getActions() { return actions; } - public void setActions(Actions actions) { this.actions = actions; } - public String getName() { return name; } - public void setName(String name) { this.name = name; } - public String getComment() { return comment; } - public void setComment(String comment) { this.comment = comment; } - public List getLink() { return link; } - public void setLink(List link) { this.link = link; } + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getAutoNumaStatus() { + return autoNumaStatus; + } + + public void setAutoNumaStatus(String autoNumaStatus) { + this.autoNumaStatus = autoNumaStatus; + } + + public Certificate getCertificate() { + return certificate; + } + + public void setCertificate(Certificate certificate) { + this.certificate = certificate; + } + + public Cpu getCpu() { + return cpu; + } + + public void setCpu(Cpu cpu) { + this.cpu = cpu; + } + + public String getExternalStatus() { + return externalStatus; + } + + public void setExternalStatus(String externalStatus) { + this.externalStatus = externalStatus; + } + + public HardwareInformation getHardwareInformation() { + return hardwareInformation; + } + + public void setHardwareInformation(HardwareInformation hardwareInformation) { + this.hardwareInformation = hardwareInformation; + } + + public String getKdumpStatus() { + return kdumpStatus; + } + + public void setKdumpStatus(String kdumpStatus) { + this.kdumpStatus = kdumpStatus; + } + + public Version getLibvirtVersion() { + return libvirtVersion; + } + + public void setLibvirtVersion(Version libvirtVersion) { + this.libvirtVersion = libvirtVersion; + } + + public String getMaxSchedulingMemory() { + return maxSchedulingMemory; + } + + public void setMaxSchedulingMemory(String maxSchedulingMemory) { + this.maxSchedulingMemory = maxSchedulingMemory; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public String getNumaSupported() { + return numaSupported; + } + + public void setNumaSupported(String numaSupported) { + this.numaSupported = numaSupported; + } + + public Os getOs() { + return os; + } + + public void setOs(Os os) { + this.os = os; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public String getReinstallationRequired() { + return reinstallationRequired; + } + + public void setReinstallationRequired(String reinstallationRequired) { + this.reinstallationRequired = reinstallationRequired; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public ApiSummary getSummary() { + return summary; + } + + public void setSummary(ApiSummary summary) { + this.summary = summary; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUpdateAvailable() { + return updateAvailable; + } + + public void setUpdateAvailable(String updateAvailable) { + this.updateAvailable = updateAvailable; + } + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } + + public String getVgpuPlacement() { + return vgpuPlacement; + } + + public void setVgpuPlacement(String vgpuPlacement) { + this.vgpuPlacement = vgpuPlacement; + } + + public Ref getCluster() { + return cluster; + } + + public void setCluster(Ref cluster) { + this.cluster = cluster; + } + + public NamedList getActions() { + return actions; + } + + public void setActions(NamedList actions) { + this.actions = actions; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java index ada443f27884..a1d4b4aa734d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java @@ -31,10 +31,27 @@ public class HostSummary { @JsonProperty("total") private String total; - public String getActive() { return active; } - public void setActive(String active) { this.active = active; } - public String getMigrating() { return migrating; } - public void setMigrating(String migrating) { this.migrating = migrating; } - public String getTotal() { return total; } - public void setTotal(String total) { this.total = total; } + public String getActive() { + return active; + } + + public void setActive(String active) { + this.active = active; + } + + public String getMigrating() { + return migrating; + } + + public void setMigrating(String migrating) { + this.migrating = migrating; + } + + public String getTotal() { + return total; + } + + public void setTotal(String total) { + this.total = total; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java deleted file mode 100644 index 17b3f77de3ef..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Hosts.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class Hosts { - @JsonProperty("host") - private List host; - - public Hosts() {} - public Hosts(List host) { this.host = host; } - - public List getHost() { return host; } - public void setHost(List host) { this.host = host; } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java index f2ff074da5ba..b0a26daa1049 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java @@ -40,7 +40,7 @@ public class ImageTransfer extends BaseDto { private Ref host; private Ref image; private Ref disk; - private Actions actions; + private NamedList actions; @JacksonXmlElementWrapper(useWrapping = false) public List link; @@ -157,11 +157,11 @@ public void setDisk(Ref disk) { this.disk = disk; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java deleted file mode 100644 index 4414846de608..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfers.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "image_transfers") -public class ImageTransfers { - @JsonProperty("image_transfer") - private List imageTransfer; - - public List getImageTransfer() { - return imageTransfer; - } - - public void setImageTransfer(List imageTransfer) { - this.imageTransfer = imageTransfer; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java deleted file mode 100644 index 11d94cc41791..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ips.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class Ips { - - @JacksonXmlElementWrapper(useWrapping = false) - private List ip; - - public Ips(final List ip) { - this.ip = ip; - } - - public List getIp() { - return ip; - } - - public void setIp(List ip) { - this.ip = ip; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java index 43121439b501..13b0e8a02fd1 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java @@ -30,38 +30,88 @@ public class Job extends BaseDto { private Long endTime; private String status; private Ref owner; - private Actions actions; + private NamedList actions; private String description; private List link; // getters and setters - public String getAutoCleared() { return autoCleared; } - public void setAutoCleared(String autoCleared) { this.autoCleared = autoCleared; } + public String getAutoCleared() { + return autoCleared; + } - public String getExternal() { return external; } - public void setExternal(String external) { this.external = external; } + public void setAutoCleared(String autoCleared) { + this.autoCleared = autoCleared; + } - public Long getLastUpdated() { return lastUpdated; } - public void setLastUpdated(Long lastUpdated) { this.lastUpdated = lastUpdated; } + public String getExternal() { + return external; + } - public Long getStartTime() { return startTime; } - public void setStartTime(Long startTime) { this.startTime = startTime; } + public void setExternal(String external) { + this.external = external; + } - public Long getEndTime() { return endTime; } - public void setEndTime(Long endTime) { this.endTime = endTime; } + public Long getLastUpdated() { + return lastUpdated; + } - public String getStatus() { return status; } - public void setStatus(String status) { this.status = status; } + public void setLastUpdated(Long lastUpdated) { + this.lastUpdated = lastUpdated; + } - public Ref getOwner() { return owner; } - public void setOwner(Ref owner) { this.owner = owner; } + public Long getStartTime() { + return startTime; + } - public Actions getActions() { return actions; } - public void setActions(Actions actions) { this.actions = actions; } + public void setStartTime(Long startTime) { + this.startTime = startTime; + } - public String getDescription() { return description; } - public void setDescription(String description) { this.description = description; } + public Long getEndTime() { + return endTime; + } - public List getLink() { return link; } - public void setLink(List link) { this.link = link; } + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Ref getOwner() { + return owner; + } + + public void setOwner(Ref owner) { + this.owner = owner; + } + + public NamedList getActions() { + return actions; + } + + public void setActions(NamedList actions) { + this.actions = actions; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java deleted file mode 100644 index 904950ae0a7a..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Jobs.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownershjob. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class Jobs { - - @JacksonXmlElementWrapper(useWrapping = false) - private List job; - - public Jobs(final List job) { - this.job = job; - } - - public List getJob() { - return job; - } - - public void setJob(List job) { - this.job = job; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java index c040323b8d09..fb7c2aa664b3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; public class NamedList { private final String name; @@ -54,4 +55,9 @@ public static NamedList fromMap(Map> map) { Entry> e = map.entrySet().iterator().next(); return new NamedList<>(e.getKey(), e.getValue()); } + + @JsonIgnore + public List getItems() { + return items; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java index 79e84fb3b172..bb72a2ad323f 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java @@ -27,7 +27,7 @@ public class Network extends BaseDto { private String mtu; // oVirt prints as string private String portIsolation; // "false" private String stp; // "false" - private NetworkUsages usages; // { usage: ["vm"] } + private NamedList usages; // { usage: ["vm"] } private String vdsmName; private Ref dataCenter; @@ -39,37 +39,88 @@ public class Network extends BaseDto { @JsonProperty("link") private List link; - public Network() {} + public Network() { + } // ---- getters / setters ---- - public String getMtu() { return mtu; } - public void setMtu(final String mtu) { this.mtu = mtu; } + public String getMtu() { + return mtu; + } - public String getPortIsolation() { return portIsolation; } - public void setPortIsolation(final String portIsolation) { this.portIsolation = portIsolation; } + public void setMtu(final String mtu) { + this.mtu = mtu; + } - public String getStp() { return stp; } - public void setStp(final String stp) { this.stp = stp; } + public String getPortIsolation() { + return portIsolation; + } - public NetworkUsages getUsages() { return usages; } - public void setUsages(final NetworkUsages usages) { this.usages = usages; } + public void setPortIsolation(final String portIsolation) { + this.portIsolation = portIsolation; + } - public String getVdsmName() { return vdsmName; } - public void setVdsmName(final String vdsmName) { this.vdsmName = vdsmName; } + public String getStp() { + return stp; + } - public Ref getDataCenter() { return dataCenter; } - public void setDataCenter(final Ref dataCenter) { this.dataCenter = dataCenter; } + public void setStp(final String stp) { + this.stp = stp; + } - public String getName() { return name; } - public void setName(final String name) { this.name = name; } + public NamedList getUsages() { + return usages; + } - public String getDescription() { return description; } - public void setDescription(final String description) { this.description = description; } + public void setUsages(final NamedList usages) { + this.usages = usages; + } - public String getComment() { return comment; } - public void setComment(final String comment) { this.comment = comment; } + public String getVdsmName() { + return vdsmName; + } - public List getLink() { return link; } - public void setLink(final List link) { this.link = link; } + public void setVdsmName(final String vdsmName) { + this.vdsmName = vdsmName; + } + + public Ref getDataCenter() { + return dataCenter; + } + + public void setDataCenter(final Ref dataCenter) { + this.dataCenter = dataCenter; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getComment() { + return comment; + } + + public void setComment(final String comment) { + this.comment = comment; + } + + public List getLink() { + return link; + } + + public void setLink(final List link) { + this.link = link; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java deleted file mode 100644 index da5e1c2aeec5..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NetworkUsages.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class NetworkUsages { - private List usage; - - public NetworkUsages() { - } - - public NetworkUsages(final List usage) { - this.usage = usage; - } - - public List getUsage() { - return usage; - } - - public void setUsage(final List usage) { - this.usage = usage; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java deleted file mode 100644 index 9b96b6e8c2d1..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Networks.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class Networks { - @JsonProperty("network") - private List network; - - public Networks() {} - public Networks(List network) { this.network = network; } - - public List getNetwork() { return network; } - public void setNetwork(List network) { this.network = network; } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java index 0b0a9043e513..2f866abef7f4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java @@ -35,7 +35,7 @@ public class Nic extends BaseDto { public String synced; private Ref vnicProfile; private Vm vm; - private ReportedDevices reportedDevices; + private NamedList reportedDevices; public Nic() { } @@ -112,11 +112,11 @@ public void setVm(Vm vm) { this.vm = vm; } - public ReportedDevices getReportedDevices() { + public NamedList getReportedDevices() { return reportedDevices; } - public void setReportedDevices(ReportedDevices reportedDevices) { + public void setReportedDevices(NamedList reportedDevices) { this.reportedDevices = reportedDevices; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java index 37c0259fa53c..1d1a46675015 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java @@ -32,7 +32,8 @@ public final class Nics { @JacksonXmlElementWrapper(useWrapping = false) public List nic; - public Nics() {} + public Nics() { + } public Nics(final List nic) { this.nic = nic; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java index 49011b303dbc..a925d6ec4450 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java @@ -20,7 +20,7 @@ public class ReportedDevice extends BaseDto { private String comment; private String description; - private Ips ips; + private NamedList ips; private Mac Mac; private String name; private String type; @@ -42,11 +42,11 @@ public void setDescription(String description) { this.description = description; } - public Ips getIps() { + public NamedList getIps() { return ips; } - public void setIps(Ips ips) { + public void setIps(NamedList ips) { this.ips = ips; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java deleted file mode 100644 index 7348b0ca6fa9..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevices.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ReportedDevices { - - @JacksonXmlElementWrapper(useWrapping = false) - private List reportedDevice; - - public ReportedDevices(final List reportedDevice) { - this.reportedDevice = reportedDevice; - } - - public List getReportedDevice() { - return reportedDevice; - } - - public void setReportedDevice(List reportedDevice) { - this.reportedDevice = reportedDevice; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java index 218a9d227d11..616e6317d90d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java @@ -30,13 +30,14 @@ public class Snapshot extends BaseDto { private String persistMemorystate; private String snapshotStatus; private String snapshotType; - private Actions actions; + private NamedList actions; private String description; @JacksonXmlElementWrapper(useWrapping = false) private List link; private Vm vm; - public Snapshot() {} + public Snapshot() { + } public Long getDate() { return date; @@ -70,11 +71,11 @@ public void setSnapshotType(final String snapshotType) { this.snapshotType = snapshotType; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(final Actions actions) { + public void setActions(final NamedList actions) { this.actions = actions; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java deleted file mode 100644 index 66a9b93e46d6..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshots.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "snapshots") -public final class Snapshots { - - @JsonProperty("snapshot") - @JacksonXmlElementWrapper(useWrapping = false) - public List snapshot; - - public Snapshots() {} - - public Snapshots(final List snapshot) { - this.snapshot = snapshot; - } -} - diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java index 9dfadd73e0d6..fff9d5f75ce8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java @@ -45,8 +45,8 @@ public final class StorageDomain extends BaseDto { private String supportsDiscard; private String supportsDiscardZeroesData; private Storage storage; - private DataCenters dataCenters; - private Actions actions; + private NamedList dataCenters; + private NamedList actions; @JacksonXmlElementWrapper(useWrapping = false) private List link; @@ -210,19 +210,19 @@ public void setStorage(Storage storage) { this.storage = storage; } - public DataCenters getDataCenters() { + public NamedList getDataCenters() { return dataCenters; } - public void setDataCenters(DataCenters dataCenters) { + public void setDataCenters(NamedList dataCenters) { this.dataCenters = dataCenters; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java deleted file mode 100644 index 644986998c40..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomains.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "storage_domains") -public final class StorageDomains { - - @JsonProperty("storage_domain") - @JacksonXmlElementWrapper(useWrapping = false) - private List storageDomain; - - public List getStorageDomain() { - return storageDomain; - } - - public void setStorageDomain(List storageDomain) { - this.storageDomain = storageDomain; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index 04ba3f99eda9..667eb7d00b11 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -28,7 +28,8 @@ public final class Version { private String minor; private String revision; - public Version() {} + public Version() { + } public String getBuild() { return build; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 5c6fdf21a1f6..9d18dcc22346 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -48,11 +48,11 @@ public final class Vm extends BaseDto { private String stateless; // true|false private String type; // "server" private String origin; // "ovirt" - private Actions actions; // actions.link[] + private NamedList actions; // actions.link[] @JacksonXmlElementWrapper(useWrapping = false) private List link; // related resources private EmptyElement tags; // empty - private DiskAttachments diskAttachments; + private NamedList diskAttachments; private Nics nics; private VmInitialization initialization; @@ -200,11 +200,11 @@ public void setOrigin(String origin) { this.origin = origin; } - public Actions getActions() { + public NamedList getActions() { return actions; } - public void setActions(Actions actions) { + public void setActions(NamedList actions) { this.actions = actions; } @@ -224,11 +224,11 @@ public void setTags(EmptyElement tags) { this.tags = tags; } - public DiskAttachments getDiskAttachments() { + public NamedList getDiskAttachments() { return diskAttachments; } - public void setDiskAttachments(DiskAttachments diskAttachments) { + public void setDiskAttachments(NamedList diskAttachments) { this.diskAttachments = diskAttachments; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java deleted file mode 100644 index df981129f1ce..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vms.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -/** - * Required list response: - * { "vm": [ {..}, {..} ] } - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ "vm" }) -@JacksonXmlRootElement(localName = "vms") -public final class Vms { - @JsonProperty("vm") - @JacksonXmlElementWrapper(useWrapping = false) - public List vm; - - public Vms() {} - - public Vms(final List vm) { - this.vm = vm; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java deleted file mode 100644 index d528e946bf6e..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfiles.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Root container for /ovirt-engine/api/vnicprofiles - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class VnicProfiles { - - @JsonProperty("vnic_profile") - private List vnicProfile; - - public VnicProfiles() { - } - - public VnicProfiles(final List vnicProfile) { - this.vnicProfile = vnicProfile; - } - - public List getVnicProfile() { - return vnicProfile; - } - - public void setVnicProfile(final List vnicProfile) { - this.vnicProfile = vnicProfile; - } -} From 30136c814a3c4357ed54a9737d314b63353032ac Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 15 Feb 2026 20:10:22 +0530 Subject: [PATCH 042/129] Image server on kvm host - with image_server.py http server --- .../org/apache/cloudstack/backup/Backup.java | 2 - .../cloudstack/backup/ImageTransfer.java | 2 - .../backup/CreateImageTransferCommand.java | 24 +- .../backup/FinalizeImageTransferCommand.java | 14 +- .../cloudstack/backup/StartBackupCommand.java | 16 +- .../backup/StartNBDServerCommand.java | 14 +- .../backup/StopNBDServerCommand.java | 8 +- .../apache/cloudstack/backup/BackupVO.java | 12 - .../cloudstack/backup/ImageTransferVO.java | 17 +- .../backup/dao/ImageTransferDao.java | 1 - .../backup/dao/ImageTransferDaoImpl.java | 12 - .../META-INF/db/schema-42100to42200.sql | 4 - .../META-INF/db/schema-42210to42300.sql | 3 +- .../resource/LibvirtComputingResource.java | 10 + ...virtCreateImageTransferCommandWrapper.java | 134 +- ...rtFinalizeImageTransferCommandWrapper.java | 110 ++ .../LibvirtStartBackupCommandWrapper.java | 18 +- .../LibvirtStartNBDServerCommandWrapper.java | 37 +- .../LibvirtStopNBDServerCommandWrapper.java | 4 +- scripts/vm/hypervisor/kvm/image_server.py | 1520 +++++++++++++++++ .../backup/IncrementalBackupServiceImpl.java | 98 +- .../resource/NfsSecondaryStorageResource.java | 216 --- 22 files changed, 1859 insertions(+), 417 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java create mode 100644 scripts/vm/hypervisor/kvm/image_server.py diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index bc464beeb6d8..42afc7f196ce 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -38,8 +38,6 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Long getHostId(); - Integer getNbdPort(); - enum Status { Allocated, Queued, BackingUp, ReadyForTransfer, FinalizingTransfer, BackedUp, Error, Failed, Restoring, Removed, Expunged } diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java index cf09749bcfc6..f7fe1e9c2bb2 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -49,8 +49,6 @@ public enum Phase { long getHostId(); - int getNbdPort(); - String getTransferUrl(); Phase getPhase(); diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 4fb8743b6252..3e042bf42491 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -21,9 +21,8 @@ public class CreateImageTransferCommand extends Command { private String transferId; - private String hostIpAddress; private String exportName; - private int nbdPort; + private String socket; private String direction; private String checkpointId; private String file; @@ -32,22 +31,21 @@ public class CreateImageTransferCommand extends Command { public CreateImageTransferCommand() { } - private CreateImageTransferCommand(String transferId, String hostIpAddress, String direction) { + private CreateImageTransferCommand(String transferId, String direction, String socket) { this.transferId = transferId; - this.hostIpAddress = hostIpAddress; this.direction = direction; + this.socket = socket; } - public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String exportName, int nbdPort, String checkpointId) { - this(transferId, hostIpAddress, direction); + public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId) { + this(transferId, direction, socket); this.backend = ImageTransfer.Backend.nbd; this.exportName = exportName; - this.nbdPort = nbdPort; this.checkpointId = checkpointId; } - public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String file) { - this(transferId, hostIpAddress, direction); + public CreateImageTransferCommand(String transferId, String direction, String socket, String file) { + this(transferId, direction, socket); if (direction == ImageTransfer.Direction.download.toString()) { throw new IllegalArgumentException("File backend is only supported for upload"); } @@ -59,8 +57,8 @@ public String getExportName() { return exportName; } - public int getNbdPort() { - return nbdPort; + public String getSocket() { + return socket; } public String getFile() { @@ -71,10 +69,6 @@ public ImageTransfer.Backend getBackend() { return backend; } - public String getHostIpAddress() { - return hostIpAddress; - } - public String getTransferId() { return transferId; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java index f1a0285ef6ed..84d9b1ff8186 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -21,30 +21,18 @@ public class FinalizeImageTransferCommand extends Command { private String transferId; - private String direction; - private int nbdPort; public FinalizeImageTransferCommand() { } - public FinalizeImageTransferCommand(String transferId, String direction, int nbdPort) { + public FinalizeImageTransferCommand(String transferId) { this.transferId = transferId; - this.direction = direction; - this.nbdPort = nbdPort; } public String getTransferId() { return transferId; } - public int getNbdPort() { - return nbdPort; - } - - public String getDirection() { - return direction; - } - @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index b43c46618435..0fc7d4e26b33 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -26,23 +26,21 @@ public class StartBackupCommand extends Command { private String toCheckpointId; private String fromCheckpointId; private Long fromCheckpointCreateTime; - private int nbdPort; + private String socket; private Map diskPathUuidMap; - private String hostIpAddress; private boolean stoppedVM; public StartBackupCommand() { } public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, - int nbdPort, Map diskPathUuidMap, String hostIpAddress, boolean stoppedVM) { + String socket, Map diskPathUuidMap, boolean stoppedVM) { this.vmName = vmName; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; this.fromCheckpointCreateTime = fromCheckpointCreateTime; - this.nbdPort = nbdPort; + this.socket = socket; this.diskPathUuidMap = diskPathUuidMap; - this.hostIpAddress = hostIpAddress; this.stoppedVM = stoppedVM; } @@ -62,8 +60,8 @@ public Long getFromCheckpointCreateTime() { return fromCheckpointCreateTime; } - public int getNbdPort() { - return nbdPort; + public String getSocket() { + return socket; } public Map getDiskPathUuidMap() { @@ -74,10 +72,6 @@ public boolean isIncremental() { return fromCheckpointId != null && !fromCheckpointId.isEmpty(); } - public String getHostIpAddress() { - return hostIpAddress; - } - public boolean isStoppedVM() { return stoppedVM; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java index 887937ffb4c8..b0e452df33c6 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -24,27 +24,31 @@ public class StartNBDServerCommand extends Command { private String hostIpAddress; private String exportName; private String volumePath; - private int nbdPort; + private String socket; private String direction; public StartNBDServerCommand() { } - public StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, int nbdPort, String direction) { + protected StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, String direction) { this.transferId = transferId; this.hostIpAddress = hostIpAddress; this.exportName = exportName; this.volumePath = volumePath; - this.nbdPort = nbdPort; this.direction = direction; } + public StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, String socket, String direction) { + this(transferId, hostIpAddress, exportName, volumePath, direction); + this.socket = socket; + } + public String getExportName() { return exportName; } - public int getNbdPort() { - return nbdPort; + public String getSocket() { + return socket; } public String getHostIpAddress() { diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java index 4f2b6401480e..d75168a22eb2 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java @@ -22,25 +22,19 @@ public class StopNBDServerCommand extends Command { private String transferId; private String direction; - private int nbdPort; public StopNBDServerCommand() { } - public StopNBDServerCommand(String transferId, String direction, int nbdPort) { + public StopNBDServerCommand(String transferId, String direction) { this.transferId = transferId; this.direction = direction; - this.nbdPort = nbdPort; } public String getTransferId() { return transferId; } - public int getNbdPort() { - return nbdPort; - } - public String getDirection() { return direction; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 4705cd0159bd..d589f9e6bef8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -115,9 +115,6 @@ public class BackupVO implements Backup { @Column(name = "host_id") private Long hostId; - @Column(name = "nbd_port") - private Integer nbdPort; - @Transient Map details; @@ -339,13 +336,4 @@ public Long getHostId() { public void setHostId(Long hostId) { this.hostId = hostId; } - - @Override - public Integer getNbdPort() { - return nbdPort; - } - - public void setNbdPort(Integer nbdPort) { - this.nbdPort = nbdPort; - } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 6562ba74a777..c391eae2e86b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -51,8 +51,8 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "host_id") private long hostId; - @Column(name = "nbd_port") - private int nbdPort; + @Column(name = "socket") + private String socket; @Column(name = "file") private String file; @@ -114,10 +114,10 @@ private ImageTransferVO(String uuid, long diskId, long hostId, Phase phase, Dire this.created = new Date(); } - public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId); this.backupId = backupId; - this.nbdPort = nbdPort; + this.socket = socket; this.backend = Backend.nbd; } @@ -164,13 +164,8 @@ public void setHostId(long hostId) { this.hostId = hostId; } - @Override - public int getNbdPort() { - return nbdPort; - } - - public void setNbdPort(int nbdPort) { - this.nbdPort = nbdPort; + public void setSocket(String socket) { + this.socket = socket; } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index e8c30d27ee79..e71dffb22d56 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -27,7 +27,6 @@ public interface ImageTransferDao extends GenericDao { List listByBackupId(Long backupId); ImageTransferVO findByUuid(String uuid); - ImageTransferVO findByNbdPort(int port); ImageTransferVO findByVolume(Long volumeId); ImageTransferVO findUnfinishedByVolume(Long volumeId); List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 7e311d2a00fe..95741fa054d1 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -34,7 +34,6 @@ public class ImageTransferDaoImpl extends GenericDaoBase private SearchBuilder backupIdSearch; private SearchBuilder uuidSearch; - private SearchBuilder nbdPortSearch; private SearchBuilder volumeSearch; private SearchBuilder volumeUnfinishedSearch; private SearchBuilder phaseDirectionSearch; @@ -52,10 +51,6 @@ protected void init() { uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); uuidSearch.done(); - nbdPortSearch = createSearchBuilder(); - nbdPortSearch.and("nbdPort", nbdPortSearch.entity().getNbdPort(), SearchCriteria.Op.EQ); - nbdPortSearch.done(); - volumeSearch = createSearchBuilder(); volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ); volumeSearch.done(); @@ -85,13 +80,6 @@ public ImageTransferVO findByUuid(String uuid) { return findOneBy(sc); } - @Override - public ImageTransferVO findByNbdPort(int port) { - SearchCriteria sc = nbdPortSearch.create(); - sc.setParameters("nbdPort", port); - return findOneBy(sc); - } - @Override public ImageTransferVO findByVolume(Long volumeId) { SearchCriteria sc = volumeSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 1e2654213870..044f7475324f 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -92,7 +92,3 @@ CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.counter', 'uc_counter__provider_ UPDATE `cloud`.`configuration` SET `scope` = 2 WHERE `name` = 'use.https.to.upload'; -- Delete the configuration for 'use.https.to.upload' from StoragePool DELETE FROM `cloud`.`storage_pool_details` WHERE `name` = 'use.https.to.upload'; - -<<<<<<< HEAD -======= ->>>>>>> 1ec4e52fa6 (Support file backend for cow format: api and server) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index f81e2904841f..b0063bff53e0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -123,7 +123,6 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VAR CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for this backup session"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'nbd_port', 'INT DEFAULT NULL COMMENT "NBD server port for backup"'); -- Add checkpoint tracking fields to vm_instance table for domain recreation CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Active checkpoint id tracked for incremental backups"'); @@ -139,10 +138,10 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( `backup_id` bigint unsigned COMMENT 'Backup ID', `disk_id` bigint unsigned NOT NULL COMMENT 'Disk/Volume ID', `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', - `nbd_port` int NOT NULL COMMENT 'NBD port', `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', `file` varchar(255) COMMENT 'File for the file backend', `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', + `socket` varchar(255) COMMENT 'Unix socket for nbd backend', `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', `backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file', `progress` int COMMENT 'Transfer progress percentage (0-100)', diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dc137376f7c6..dfba9ad11157 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -395,6 +395,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String heartBeatPath; private String vmActivityCheckPath; private String nasBackupPath; + private String imageServerPath; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -809,6 +810,10 @@ public String getNasBackupPath() { return nasBackupPath; } + public String getImageServerPath() { + return imageServerPath; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -1095,6 +1100,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } + imageServerPath = Script.findScript(kvmScriptsDir, "image_server.py"); + if (imageServerPath == null) { + throw new ConfigurationException("Unable to find image_server.py"); + } + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 1db594d169fb..d3eca1aeb23f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -17,8 +17,16 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -26,36 +34,128 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; +import com.google.gson.GsonBuilder; @ResourceWrapper(handles = CreateImageTransferCommand.class) public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); - private CreateImageTransferAnswer handleUpload(CreateImageTransferCommand cmd) { - return new CreateImageTransferAnswer(cmd, false, "Image Upload is not handled by KVM agent"); - } + private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputingResource resource) { + final String imageServerScript = resource.getImageServerPath(); + String unitName = "cloudstack-image-server"; - private CreateImageTransferAnswer handleDownload(CreateImageTransferCommand cmd) { - String exportName = cmd.getExportName(); - int nbdPort = cmd.getNbdPort(); - try { - String hostIpAddress = cmd.getHostIpAddress(); - String transferUrl = String.format("nbd://%s:%d/%s", hostIpAddress, nbdPort, exportName); + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return true; + } + + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port %d", + unitName, imageServerScript, imageServerPort); - return new CreateImageTransferAnswer(cmd, true, "Image transfer created for download", - cmd.getTransferId(), transferUrl); + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); - } catch (Exception e) { - return new CreateImageTransferAnswer(cmd, false, "Error creating image transfer: " + e.getMessage()); + if (startResult != null) { + logger.error(String.format("Failed to start the Image server: %s", startResult)); + return false; } + + // Wait with timeout until the service is up + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("Image server is now active (attempt %d)", attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + if (!serviceActive) { + logger.error(String.format("Image server failed to start within %d seconds", maxWaitSeconds)); + return false; + } + + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + String.format("Error in opening up image server port %d", imageServerPort)); + + return true; } - @Override public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { - if (cmd.getDirection().equals("download")) { - return handleDownload(cmd); + final String transferId = cmd.getTransferId(); + ImageTransfer.Backend backend = cmd.getBackend(); + + if (StringUtils.isBlank(transferId)) { + return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); + } + + final Map payload = new HashMap<>(); + payload.put("backend", backend.toString()); + + if (backend == ImageTransfer.Backend.file) { + final String filePath = cmd.getFile(); + if (StringUtils.isBlank(filePath)) { + return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); + } + payload.put("file", filePath); } else { - return handleUpload(cmd); + String socket = cmd.getSocket(); + final String exportName = cmd.getExportName(); + if (StringUtils.isBlank(socket)) { + return new CreateImageTransferAnswer(cmd, false, "Empty socket."); + } + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); + } + payload.put("socket", "/tmp/imagetransfer/" + socket + ".sock"); + payload.put("export", exportName); + String checkpointId = cmd.getCheckpointId(); + if (checkpointId != null) { + payload.put("export_bitmap", exportName + "-" + checkpointId.substring(0, 4)); + } } + + try { + final String json = new GsonBuilder().create().toJson(payload); + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + final File transferFile = new File("/tmp/imagetransfer", transferId); + FileUtils.writeStringToFile(transferFile, json, "UTF-8"); + + } catch (IOException e) { + logger.warn("Failed to prepare image transfer on KVM host", e); + return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on KVM host: " + e.getMessage()); + } + + final int imageServerPort = 54323; + startImageServerIfNotRunning(imageServerPort, resource); + + final String transferUrl = String.format("http://%s:%d/images/%s", resource.getPrivateIp(), imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java new file mode 100644 index 000000000000..c2c9d7a797d6 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -0,0 +1,110 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; + +import org.apache.cloudstack.backup.FinalizeImageTransferCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = FinalizeImageTransferCommand.class) +public class LibvirtFinalizeImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private boolean stopImageServer() { + String unitName = "cloudstack-image-server"; + final int imageServerPort = 54323; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + logger.info(String.format("Image server not running, resetting failed state")); + resetService(unitName); + // Still try to remove firewall rule in case it exists + removeFirewallRule(imageServerPort); + return true; + } + + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + logger.info(String.format("Image server %s stopped", unitName)); + + removeFirewallRule(imageServerPort); + + return true; + } + + private void removeFirewallRule(int port) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); + Script removeScript = new Script("/bin/bash", logger); + removeScript.add("-c"); + removeScript.add(String.format("iptables -D INPUT %s || true", rule)); + String result = removeScript.execute(); + if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { + logger.debug(String.format("Firewall rule removal result for port %d: %s", port, result)); + } else { + logger.info(String.format("Firewall rule removed for port %d (or did not exist)", port)); + } + } + + public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + if (StringUtils.isBlank(transferId)) { + return new Answer(cmd, false, "transferId is empty."); + } + + final File transferFile = new File("/tmp/imagetransfer", transferId); + if (transferFile.exists() && !transferFile.delete()) { + return new Answer(cmd, false, "Failed to delete transfer config file: " + transferFile.getAbsolutePath()); + } + + try (Stream stream = Files.list(Paths.get("/tmp/imagetransfer"))) { + if (!stream.findAny().isPresent()) { + stopImageServer(); + } + } catch (IOException e) { + logger.warn("Failed to list /tmp/imagetransfer", e); + } + + return new Answer(cmd, true, "Image transfer finalized."); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index bc3faa044933..04416559c578 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -39,7 +39,7 @@ public class LibvirtStartBackupCommandWrapper extends CommandWrapper\n"); @@ -149,7 +154,8 @@ private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, xml.append(" ").append(fromCheckpointId).append("\n"); } - xml.append(String.format(" \n", cmd.getHostIpAddress(), nbdPort)); + xml.append(String.format(" \n", socket)); + xml.append(" \n"); Map diskPathUuidMap = cmd.getDiskPathUuidMap(); @@ -185,7 +191,7 @@ private String createCheckpointXml(String checkpointId) { ""; } - private Answer handleStoppedVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource, String toCheckpointId) { + private Answer handleStoppedVmBackup(StartBackupCommand cmd, String toCheckpointId) { String vmName = cmd.getVmName(); Map diskPathUuidMap = cmd.getDiskPathUuidMap(); for (Map.Entry entry : diskPathUuidMap.entrySet()) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index 7a8588809df6..71d9a06a3605 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; + import org.apache.cloudstack.backup.StartNBDServerAnswer; import org.apache.cloudstack.backup.StartNBDServerCommand; import org.apache.logging.log4j.Logger; @@ -26,6 +28,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; @ResourceWrapper(handles = StartNBDServerCommand.class) @@ -35,22 +38,25 @@ public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper backend mapping: +# CloudStack writes a JSON file at /tmp/imagetransfer/ with: +# - NBD backend: {"backend": "nbd", "socket": "/tmp/imagetransfer/.sock", "export": "vda", "export_bitmap": "..."} +# - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} +# +# This server reads that file on-demand. +_CFG_DIR = "/tmp/imagetransfer" +_CFG_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {} +_CFG_CACHE_GUARD = threading.Lock() + + +def _json_bytes(obj: Any) -> bytes: + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _merge_dirty_zero_extents( + allocation_extents: List[Tuple[int, int, bool]], + dirty_extents: List[Tuple[int, int, bool]], + size: int, +) -> List[Dict[str, Any]]: + """ + Merge allocation (start, length, zero) and dirty (start, length, dirty) extents + into a single list of {start, length, dirty, zero} with unified boundaries. + """ + boundaries: Set[int] = {0, size} + for start, length, _ in allocation_extents: + boundaries.add(start) + boundaries.add(start + length) + for start, length, _ in dirty_extents: + boundaries.add(start) + boundaries.add(start + length) + sorted_boundaries = sorted(boundaries) + + def lookup( + extents: List[Tuple[int, int, bool]], offset: int, default: bool + ) -> bool: + for start, length, flag in extents: + if start <= offset < start + length: + return flag + return default + + result: List[Dict[str, Any]] = [] + for i in range(len(sorted_boundaries) - 1): + a, b = sorted_boundaries[i], sorted_boundaries[i + 1] + if a >= b: + continue + result.append( + { + "start": a, + "length": b - a, + "dirty": lookup(dirty_extents, a, False), + "zero": lookup(allocation_extents, a, False), + } + ) + return result + + +def _is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: + """True if extents is the single-extent fallback (dirty=false, zero=false).""" + return ( + len(extents) == 1 + and extents[0].get("dirty") is False + and extents[0].get("zero") is False + ) + + +def _get_image_lock(image_id: str) -> threading.Lock: + with _IMAGE_LOCKS_GUARD: + lock = _IMAGE_LOCKS.get(image_id) + if lock is None: + lock = threading.Lock() + _IMAGE_LOCKS[image_id] = lock + return lock + + +def _now_s() -> float: + return time.monotonic() + + +def _safe_transfer_id(image_id: str) -> Optional[str]: + """ + Only allow a single filename component to avoid path traversal. + We intentionally keep validation simple: reject anything containing '/' or '\\'. + """ + if not image_id: + return None + if image_id != os.path.basename(image_id): + return None + if "/" in image_id or "\\" in image_id: + return None + if image_id in (".", ".."): + return None + return image_id + + +def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: + safe_id = _safe_transfer_id(image_id) + if safe_id is None: + return None + + cfg_path = os.path.join(_CFG_DIR, safe_id) + try: + st = os.stat(cfg_path) + except FileNotFoundError: + return None + except OSError as e: + logging.error("cfg stat failed image_id=%s err=%r", image_id, e) + return None + + with _CFG_CACHE_GUARD: + cached = _CFG_CACHE.get(safe_id) + if cached is not None: + cached_mtime, cached_cfg = cached + # Use cached config if the file hasn't changed. + if float(st.st_mtime) == float(cached_mtime): + return cached_cfg + + try: + with open(cfg_path, "rb") as f: + raw = f.read(4096) + except OSError as e: + logging.error("cfg read failed image_id=%s err=%r", image_id, e) + return None + + try: + obj = json.loads(raw.decode("utf-8")) + except Exception as e: + logging.error("cfg parse failed image_id=%s err=%r", image_id, e) + return None + + if not isinstance(obj, dict): + logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) + return None + + backend = obj.get("backend") + if backend is None: + backend = "nbd" + if not isinstance(backend, str): + logging.error("cfg invalid backend type image_id=%s", image_id) + return None + backend = backend.lower() + if backend not in ("nbd", "file"): + logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) + return None + + if backend == "file": + file_path = obj.get("file") + if not isinstance(file_path, str) or not file_path.strip(): + logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) + return None + cfg = {"backend": "file", "file": file_path.strip()} + else: + socket_path = obj.get("socket") + export = obj.get("export") + export_bitmap = obj.get("export_bitmap") + if not isinstance(socket_path, str) or not socket_path.strip(): + logging.error("cfg missing/invalid socket path for nbd backend image_id=%s", image_id) + return None + socket_path = socket_path.strip() + if export is not None and (not isinstance(export, str) or not export): + logging.error("cfg missing/invalid export image_id=%s", image_id) + return None + cfg = { + "backend": "nbd", + "socket": socket_path, + "export": export, + "export_bitmap": export_bitmap, + } + + with _CFG_CACHE_GUARD: + _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) + return cfg + + +class _NbdConn: + """ + Small helper to connect to NBD over a Unix socket. + Opens a fresh handle per request, per POC requirements. + """ + + def __init__( + self, + socket_path: str, + export: Optional[str], + need_block_status: bool = False, + extra_meta_contexts: Optional[List[str]] = None, + ): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(socket_path) + self._nbd = nbd.NBD() + + # Select export name if supported/needed. + if export and hasattr(self._nbd, "set_export_name"): + self._nbd.set_export_name(export) + + # Request meta contexts before connect (for block status / dirty bitmap). + if need_block_status and hasattr(self._nbd, "add_meta_context"): + for ctx in ["base:allocation"] + (extra_meta_contexts or []): + try: + self._nbd.add_meta_context(ctx) + except Exception as e: + logging.warning("add_meta_context %r failed: %r", ctx, e) + + self._connect_existing_socket(self._sock) + + def _connect_existing_socket(self, sock: socket.socket) -> None: + # Requirement: attach libnbd to an existing socket / FD (no qemu-nbd). + # libnbd python API varies slightly by version, so try common options. + last_err: Optional[BaseException] = None + if hasattr(self._nbd, "connect_socket"): + try: + self._nbd.connect_socket(sock) + return + except Exception as e: # pragma: no cover (depends on binding) + last_err = e + try: + self._nbd.connect_socket(sock.fileno()) + return + except Exception as e2: # pragma: no cover + last_err = e2 + if hasattr(self._nbd, "connect_fd"): + try: + self._nbd.connect_fd(sock.fileno()) + return + except Exception as e: # pragma: no cover + last_err = e + raise RuntimeError( + "Unable to connect libnbd using existing socket/fd; " + f"binding missing connect_socket/connect_fd or call failed: {last_err!r}" + ) + + def size(self) -> int: + return int(self._nbd.get_size()) + + def get_capabilities(self) -> Dict[str, bool]: + """ + Query NBD export capabilities (read_only, can_flush, can_zero) from the + server handshake. Returns dict with keys read_only, can_flush, can_zero. + Uses getattr for binding name variations (is_read_only/get_read_only, etc.). + """ + out: Dict[str, bool] = { + "read_only": True, + "can_flush": False, + "can_zero": False, + } + for name, keys in [ + ("read_only", ("is_read_only", "get_read_only")), + ("can_flush", ("can_flush", "get_can_flush")), + ("can_zero", ("can_zero", "get_can_zero")), + ]: + for attr in keys: + if hasattr(self._nbd, attr): + try: + val = getattr(self._nbd, attr)() + out[name] = bool(val) + except Exception: + pass + break + return out + + def pread(self, length: int, offset: int) -> bytes: + # Expected signature: pread(length, offset) + try: + return self._nbd.pread(length, offset) + except TypeError: # pragma: no cover (binding differences) + return self._nbd.pread(offset, length) + + def pwrite(self, buf: bytes, offset: int) -> None: + # Expected signature: pwrite(buf, offset) + try: + self._nbd.pwrite(buf, offset) + except TypeError: # pragma: no cover (binding differences) + self._nbd.pwrite(offset, buf) + + def pzero(self, offset: int, size: int) -> None: + """ + Zero a byte range. Uses NBD WRITE_ZEROES when available (efficient/punch hole), + otherwise falls back to writing zero bytes via pwrite. + """ + if size <= 0: + return + # Try libnbd pwrite_zeros / zero; argument order varies by binding. + for name in ("pwrite_zeros", "zero"): + if not hasattr(self._nbd, name): + continue + fn = getattr(self._nbd, name) + try: + fn(size, offset) + return + except TypeError: + try: + fn(offset, size) + return + except TypeError: + pass + # Fallback: write zeros in chunks. + remaining = size + pos = offset + zero_buf = b"\x00" * min(CHUNK_SIZE, size) + while remaining > 0: + chunk = min(len(zero_buf), remaining) + self.pwrite(zero_buf[:chunk], pos) + pos += chunk + remaining -= chunk + + def flush(self) -> None: + if hasattr(self._nbd, "flush"): + self._nbd.flush() + return + if hasattr(self._nbd, "fsync"): + self._nbd.fsync() + return + raise RuntimeError("libnbd binding has no flush/fsync method") + + def get_zero_extents(self) -> List[Dict[str, Any]]: + """ + Query NBD block status (base:allocation) and return extents that are + hole or zero in imageio format: [{"start": ..., "length": ..., "zero": true}, ...]. + Returns [] if block status is not supported; fallback to one full-image + zero extent when we have size but block status fails. + """ + size = self.size() + if size == 0: + return [] + + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + logging.error("get_zero_extents: no block_status/block_status_64") + return self._fallback_zero_extent(size) + if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( + "base:allocation" + ): + logging.error( + "get_zero_extents: server did not negotiate base:allocation" + ) + return self._fallback_zero_extent(size) + + zero_extents: List[Dict[str, Any]] = [] + chunk = min(size, 64 * 1024 * 1024) # 64 MiB + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + # Binding typically passes (metacontext, offset, entries[, nr_entries][, error]). + metacontext = None + off = 0 + entries = None + if len(args) >= 3: + metacontext, off, entries = args[0], args[1], args[2] + else: + for a in args: + if isinstance(a, str): + metacontext = a + elif isinstance(a, int): + off = a + elif a is not None and hasattr(a, "__iter__"): + entries = a + if metacontext != "base:allocation" or entries is None: + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + if (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0: + zero_extents.append( + {"start": current, "length": length, "zero": True} + ) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return self._fallback_zero_extent(size) + + try: + while offset < size: + count = min(chunk, size - offset) + # Try (count, offset, callback) then (offset, count, callback) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.error("get_zero_extents block_status failed: %r", e) + return self._fallback_zero_extent(size) + if not zero_extents: + return self._fallback_zero_extent(size) + return zero_extents + + def _fallback_zero_extent(self, size: int) -> List[Dict[str, Any]]: + """Return one zero extent covering the whole image when block status unavailable.""" + return [{"start": 0, "length": size, "zero": True}] + + def get_allocation_extents(self) -> List[Dict[str, Any]]: + """ + Query base:allocation and return all extents (allocated and hole/zero) + as [{"start": ..., "length": ..., "zero": bool}, ...]. + Fallback when block status unavailable: one extent with zero=False. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return [{"start": 0, "length": size, "zero": False}] + if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( + "base:allocation" + ): + return [{"start": 0, "length": size, "zero": False}] + + allocation_extents: List[Dict[str, Any]] = [] + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if metacontext != "base:allocation" or entries is None: + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 + allocation_extents.append( + {"start": current, "length": length, "zero": zero} + ) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return [{"start": 0, "length": size, "zero": False}] + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_allocation_extents block_status failed: %r", e) + return [{"start": 0, "length": size, "zero": False}] + if not allocation_extents: + return [{"start": 0, "length": size, "zero": False}] + return allocation_extents + + def get_extents_dirty_and_zero( + self, dirty_bitmap_context: str + ) -> List[Dict[str, Any]]: + """ + Query block status for base:allocation and qemu:dirty-bitmap:, + merge boundaries, and return extents with dirty and zero flags. + Format: [{"start": ..., "length": ..., "dirty": bool, "zero": bool}, ...]. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return self._fallback_dirty_zero_extents(size) + if hasattr(self._nbd, "can_meta_context"): + if not self._nbd.can_meta_context("base:allocation"): + return self._fallback_dirty_zero_extents(size) + if not self._nbd.can_meta_context(dirty_bitmap_context): + logging.warning( + "dirty bitmap context %r not negotiated", dirty_bitmap_context + ) + return self._fallback_dirty_zero_extents(size) + + allocation_extents: List[Tuple[int, int, bool]] = [] # (start, length, zero) + dirty_extents: List[Tuple[int, int, bool]] = [] # (start, length, dirty) + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if entries is None or not hasattr(entries, "__iter__"): + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + if metacontext == "base:allocation": + zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 + allocation_extents.append((current, length, zero)) + elif metacontext == dirty_bitmap_context: + dirty = (flags & _NBD_STATE_DIRTY) != 0 + dirty_extents.append((current, length, dirty)) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return self._fallback_dirty_zero_extents(size) + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_extents_dirty_and_zero block_status failed: %r", e) + return self._fallback_dirty_zero_extents(size) + return _merge_dirty_zero_extents(allocation_extents, dirty_extents, size) + + def _fallback_dirty_zero_extents(self, size: int) -> List[Dict[str, Any]]: + """One extent: whole image, dirty=false, zero=false when bitmap unavailable.""" + return [{"start": 0, "length": size, "dirty": False, "zero": False}] + + def close(self) -> None: + # Best-effort; bindings may differ. + try: + if hasattr(self._nbd, "shutdown"): + self._nbd.shutdown() + except Exception: + pass + try: + if hasattr(self._nbd, "close"): + self._nbd.close() + except Exception: + pass + try: + self._sock.close() + except Exception: + pass + + def __enter__(self) -> "_NbdConn": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +class Handler(BaseHTTPRequestHandler): + server_version = "imageio-poc/0.1" + + # Keep BaseHTTPRequestHandler from printing noisy default logs + def log_message(self, fmt: str, *args: Any) -> None: + logging.info("%s - - %s", self.address_string(), fmt % args) + + def _send_imageio_headers( + self, allowed_methods: Optional[str] = None + ) -> None: + # Include these headers for compatibility with the imageio contract. + if allowed_methods is None: + allowed_methods = "GET, PUT, OPTIONS" + self.send_header("Access-Control-Allow-Methods", allowed_methods) + self.send_header("Accept-Ranges", "bytes") + + def _send_json( + self, + status: int, + obj: Any, + allowed_methods: Optional[str] = None, + ) -> None: + body = _json_bytes(obj) + self.send_response(status) + self._send_imageio_headers(allowed_methods) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + def _send_error_json(self, status: int, message: str) -> None: + self._send_json(status, {"error": message}) + + def _send_range_not_satisfiable(self, size: int) -> None: + # RFC 7233: reply with Content-Range: bytes */ + self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) + self._send_imageio_headers() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Range", f"bytes */{size}") + body = _json_bytes({"error": "range not satisfiable"}) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + def _parse_single_range(self, range_header: str, size: int) -> Tuple[int, int]: + """ + Parse a single HTTP byte range (RFC 7233) and return (start, end_inclusive). + + Supported: + - Range: bytes=START-END + - Range: bytes=START- + - Range: bytes=-SUFFIX + + Raises ValueError for invalid headers. Caller handles 416 vs 400. + """ + if size < 0: + raise ValueError("invalid size") + if not range_header: + raise ValueError("empty Range") + if "," in range_header: + raise ValueError("multiple ranges not supported") + + prefix = "bytes=" + if not range_header.startswith(prefix): + raise ValueError("only bytes ranges supported") + spec = range_header[len(prefix) :].strip() + if "-" not in spec: + raise ValueError("invalid bytes range") + + left, right = spec.split("-", 1) + left = left.strip() + right = right.strip() + + if left == "": + # Suffix range: last N bytes. + if right == "": + raise ValueError("invalid suffix range") + try: + suffix_len = int(right, 10) + except ValueError as e: + raise ValueError("invalid suffix length") from e + if suffix_len <= 0: + raise ValueError("invalid suffix length") + if size == 0: + # Nothing to serve + raise ValueError("unsatisfiable") + if suffix_len >= size: + return 0, size - 1 + return size - suffix_len, size - 1 + + # START is present + try: + start = int(left, 10) + except ValueError as e: + raise ValueError("invalid range start") from e + if start < 0: + raise ValueError("invalid range start") + if start >= size: + raise ValueError("unsatisfiable") + + if right == "": + # START- + return start, size - 1 + + try: + end = int(right, 10) + except ValueError as e: + raise ValueError("invalid range end") from e + if end < start: + raise ValueError("unsatisfiable") + if end >= size: + end = size - 1 + return start, end + + def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: + # Returns (image_id, tail) where tail is: + # None => /images/{id} + # "extents" => /images/{id}/extents + # "flush" => /images/{id}/flush + path = self.path.split("?", 1)[0] + parts = [p for p in path.split("/") if p] + if len(parts) < 2 or parts[0] != "images": + return None, None + image_id = parts[1] + tail = parts[2] if len(parts) >= 3 else None + if len(parts) > 3: + return None, None + return image_id, tail + + def _parse_query(self) -> Dict[str, List[str]]: + """Parse query string from self.path into a dict of name -> list of values.""" + if "?" not in self.path: + return {} + query = self.path.split("?", 1)[1] + return parse_qs(query, keep_blank_values=True) + + def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: + return _load_image_cfg(image_id) + + def _is_file_backend(self, cfg: Dict[str, Any]) -> bool: + return cfg.get("backend") == "file" + + def do_OPTIONS(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + if self._is_file_backend(cfg): + # File backend: full PUT only, no range writes; GET with ranges allowed; flush supported. + allowed_methods = "GET, PUT, POST, OPTIONS" + features = ["flush"] + max_writers = MAX_PARALLEL_WRITES + response = { + "unix_socket": None, + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": max_writers, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + return + # Query NBD backend for capabilities (like nbdinfo); fall back to config. + read_only = True + can_flush = False + can_zero = False + try: + with _NbdConn( + cfg["socket"], + cfg.get("export"), + ) as conn: + caps = conn.get_capabilities() + read_only = caps["read_only"] + can_flush = caps["can_flush"] + can_zero = caps["can_zero"] + except Exception as e: + logging.warning("OPTIONS: could not query NBD capabilities: %r", e) + read_only = bool(cfg.get("read_only")) + if not read_only: + can_flush = True + can_zero = True + # Report options for this image from NBD: read-only => no PUT; only advertise supported features. + if read_only: + allowed_methods = "GET, OPTIONS" + features = ["extents"] + max_writers = 0 + else: + # PATCH: JSON (zero/flush) and Range+binary (write byte range). + allowed_methods = "GET, PUT, PATCH, OPTIONS" + features = ["extents"] + if can_zero: + features.append("zero") + if can_flush: + features.append("flush") + max_writers = MAX_PARALLEL_WRITES if not read_only else 0 + response = { + "unix_socket": None, # Not used in this implementation + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": max_writers, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + + def do_GET(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "extents": + if self._is_file_backend(cfg): + self._send_error_json( + HTTPStatus.BAD_REQUEST, "extents not supported for file backend" + ) + return + query = self._parse_query() + context = (query.get("context") or [None])[0] + self._handle_get_extents(image_id, cfg, context=context) + return + if tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + range_header = self.headers.get("Range") + self._handle_get_image(image_id, cfg, range_header) + + def do_PUT(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if self.headers.get("Range") is not None or self.headers.get("Content-Range") is not None: + self._send_error_json( + HTTPStatus.BAD_REQUEST, "Range/Content-Range not supported; full writes only" + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + self._handle_put_image(image_id, cfg, content_length) + + def do_POST(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "flush": + self._handle_post_flush(image_id, cfg) + return + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + + def do_PATCH(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + if self._is_file_backend(cfg): + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "range writes and PATCH not supported for file backend; use PUT for full upload", + ) + return + + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + range_header = self.headers.get("Range") + + # Binary PATCH: Range + body writes bytes at that range (e.g. curl -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin). + if range_header is not None and content_type != "application/json": + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") + return + self._handle_patch_range(image_id, cfg, range_header, content_length) + return + + # JSON PATCH: application/json with op (zero, flush). + if content_type != "application/json": + self._send_error_json( + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0 or content_length > 64 * 1024: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + body = self.rfile.read(content_length) + if len(body) != content_length: + self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") + return + + try: + payload = json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") + return + + if not isinstance(payload, dict): + self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") + return + + op = payload.get("op") + if op == "flush": + # Flush entire image; offset and size are ignored (per spec). + self._handle_post_flush(image_id, cfg) + return + if op != "zero": + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "unsupported op; only \"zero\" and \"flush\" are supported", + ) + return + + try: + size = int(payload.get("size")) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") + return + if size <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") + return + + offset = payload.get("offset") + if offset is None: + offset = 0 + else: + try: + offset = int(offset) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") + return + if offset < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") + return + + flush = bool(payload.get("flush", False)) + + self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) + + def _handle_get_image( + self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] + ) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _READ_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") + return + + start = _now_s() + bytes_sent = 0 + try: + logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") + if self._is_file_backend(cfg): + file_path = cfg["file"] + try: + size = os.path.getsize(file_path) + except OSError as e: + logging.error("GET file size error image_id=%s path=%s err=%r", image_id, file_path, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "failed to access file") + return + start_off = 0 + end_off_incl = size - 1 if size > 0 else -1 + status = HTTPStatus.OK + content_length = size + if range_header is not None: + try: + start_off, end_off_incl = self._parse_single_range(range_header, size) + except ValueError as e: + if str(e) == "unsatisfiable": + self._send_range_not_satisfiable(size) + return + if "unsatisfiable" in str(e): + self._send_range_not_satisfiable(size) + return + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") + return + status = HTTPStatus.PARTIAL_CONTENT + content_length = (end_off_incl - start_off) + 1 + + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(content_length)) + if status == HTTPStatus.PARTIAL_CONTENT: + self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") + self.end_headers() + + offset = start_off + end_excl = end_off_incl + 1 + with open(file_path, "rb") as f: + f.seek(offset) + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = f.read(to_read) + if not data: + break + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) + else: + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + size = conn.size() + + start_off = 0 + end_off_incl = size - 1 if size > 0 else -1 + status = HTTPStatus.OK + content_length = size + if range_header is not None: + try: + start_off, end_off_incl = self._parse_single_range(range_header, size) + except ValueError as e: + if str(e) == "unsatisfiable": + self._send_range_not_satisfiable(size) + return + if "unsatisfiable" in str(e): + self._send_range_not_satisfiable(size) + return + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") + return + status = HTTPStatus.PARTIAL_CONTENT + content_length = (end_off_incl - start_off) + 1 + + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(content_length)) + if status == HTTPStatus.PARTIAL_CONTENT: + self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") + self.end_headers() + + offset = start_off + end_excl = end_off_incl + 1 + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = conn.pread(to_read, offset) + if not data: + raise RuntimeError("backend returned empty read") + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) + except Exception as e: + # If headers already sent, we can't return JSON reliably; just log. + logging.error("GET error image_id=%s err=%r", image_id, e) + try: + if not self.wfile.closed: + self.close_connection = True + except Exception: + pass + finally: + _READ_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur + ) + + def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: int) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + bytes_written = 0 + try: + logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) + if self._is_file_backend(cfg): + file_path = cfg["file"] + remaining = content_length + with open(file_path, "wb") as f: + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {bytes_written} bytes", + ) + return + f.write(chunk) + bytes_written += len(chunk) + remaining -= len(chunk) + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + else: + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + offset = 0 + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {offset} bytes", + ) + return + conn.pwrite(chunk, offset) + offset += len(chunk) + remaining -= len(chunk) + bytes_written += len(chunk) + + # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + except Exception as e: + logging.error("PUT error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur + ) + + def _handle_get_extents( + self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None + ) -> None: + # context=dirty: return extents with dirty and zero from base:allocation + bitmap. + # Otherwise: return zero/hole extents from base:allocation only. + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = _now_s() + try: + logging.info("EXTENTS start image_id=%s context=%s", image_id, context) + if context == "dirty": + export_bitmap = cfg.get("export_bitmap") + if not export_bitmap: + # Fallback: same structure as zero extents but dirty=true for all ranges + with _NbdConn( + cfg["socket"], + cfg.get("export"), + need_block_status=True, + ) as conn: + allocation = conn.get_allocation_extents() + extents = [ + {"start": e["start"], "length": e["length"], "dirty": True, "zero": e["zero"]} + for e in allocation + ] + else: + dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}" + extra_contexts: List[str] = [dirty_bitmap_ctx] + with _NbdConn( + cfg["socket"], + cfg.get("export"), + need_block_status=True, + extra_meta_contexts=extra_contexts, + ) as conn: + extents = conn.get_extents_dirty_and_zero(dirty_bitmap_ctx) + # When bitmap not actually available, same fallback: zero structure + dirty=true + if _is_fallback_dirty_response(extents): + with _NbdConn( + cfg["socket"], + cfg.get("export"), + need_block_status=True, + ) as conn: + allocation = conn.get_allocation_extents() + extents = [ + { + "start": e["start"], + "length": e["length"], + "dirty": True, + "zero": e["zero"], + } + for e in allocation + ] + else: + with _NbdConn( + cfg["socket"], + cfg.get("export"), + need_block_status=True, + ) as conn: + extents = conn.get_zero_extents() + self._send_json(HTTPStatus.OK, extents) + except Exception as e: + logging.error("EXTENTS error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = _now_s() - start + logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = _now_s() + try: + logging.info("FLUSH start image_id=%s", image_id) + if self._is_file_backend(cfg): + file_path = cfg["file"] + with open(file_path, "rb") as f: + f.flush() + os.fsync(f.fileno()) + self._send_json(HTTPStatus.OK, {"ok": True}) + else: + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + conn.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + except Exception as e: + logging.error("FLUSH error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = _now_s() - start + logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_patch_zero( + self, + image_id: str, + cfg: Dict[str, Any], + offset: int, + size: int, + flush: bool, + ) -> None: + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + try: + logging.info( + "PATCH zero start image_id=%s offset=%d size=%d flush=%s", + image_id, offset, size, flush, + ) + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + image_size = conn.size() + if offset >= image_size: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "offset must be less than image size", + ) + return + zero_size = min(size, image_size - offset) + conn.pzero(offset, zero_size) + if flush: + conn.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + except Exception as e: + logging.error("PATCH zero error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_patch_range( + self, + image_id: str, + cfg: Dict[str, Any], + range_header: str, + content_length: int, + ) -> None: + """Write request body to the image at the byte range from Range header.""" + lock = _get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + bytes_written = 0 + try: + logging.info( + "PATCH range start image_id=%s range=%s content_length=%d", + image_id, range_header, content_length, + ) + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + image_size = conn.size() + try: + start_off, end_inclusive = self._parse_single_range( + range_header, image_size + ) + except ValueError as e: + if "unsatisfiable" in str(e).lower(): + self._send_range_not_satisfiable(image_size) + else: + self._send_error_json( + HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" + ) + return + expected_len = end_inclusive - start_off + 1 + if content_length != expected_len: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"Content-Length ({content_length}) must equal range length ({expected_len})", + ) + return + offset = start_off + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {bytes_written} bytes", + ) + return + conn.pwrite(chunk, offset) + n = len(chunk) + offset += n + remaining -= n + bytes_written += n + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + except Exception as e: + logging.error("PATCH range error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "PATCH range end image_id=%s bytes=%d duration_s=%.3f", + image_id, bytes_written, dur, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") + parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") + parser.add_argument("--port", type=int, default=54323, help="Port to listen on") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + addr = (args.listen, args.port) + httpd = ThreadingHTTPServer(addr, Handler) + logging.info("listening on http://%s:%d", args.listen, args.port) + logging.info("image configs are read from %s/", _CFG_DIR) + httpd.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index be6dcae12b85..ed44ded22805 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Timer; import java.util.TimerTask; import java.util.UUID; @@ -108,15 +107,8 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject private PrimaryDataStoreDao primaryDataStoreDao; - @Inject - EndPointSelector _epSelector; - private Timer imageTransferTimer; - private static final int NBD_PORT_RANGE_START = 10809; - private static final int NBD_PORT_RANGE_END = 10909; - private static final boolean DATAPLANE_PROXY_MODE = true; - private boolean isDummyOffering(Long backupOfferingId) { if (backupOfferingId == null) { throw new CloudRuntimeException("VM not assigned a backup offering"); @@ -174,9 +166,7 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); - int nbdPort = allocateNbdPort(); Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); - backup.setNbdPort(nbdPort); backup.setHostId(hostId); // Will be changed later if incremental was done backup.setType("FULL"); @@ -206,9 +196,8 @@ public Backup startBackup(StartBackupCmd cmd) { backup.getToCheckpointId(), backup.getFromCheckpointId(), vm.getActiveCheckpointCreateTime(), - backup.getNbdPort(), + backup.getUuid(), diskPathUuidMap, - host.getPrivateIpAddress(), vm.getState() == State.Stopped ); @@ -334,29 +323,26 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu } String transferId = UUID.randomUUID().toString(); - Host host = hostDao.findById(backup.getHostId()); + String socket = backup.getUuid(); VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); if (vm.getState() == State.Stopped) { String volumePath = getVolumePathForFileBasedBackend(volume); - startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, backup.getNbdPort()); + startNBDServer(transferId, direction, backup.getHostId(), volume.getUuid(), volumePath); + socket = transferId; } CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( transferId, - host.getPrivateIpAddress(), direction, volume.getUuid(), - backup.getNbdPort(), + socket, backup.getFromCheckpointId()); try { CreateImageTransferAnswer answer; if (dummyOffering) { answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda"); - } else if (DATAPLANE_PROXY_MODE) { - EndPoint ssvm = _epSelector.findSsvm(backup.getZoneId()); - answer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); } else { answer = (CreateImageTransferAnswer) agentManager.send(backup.getHostId(), transferCmd); } @@ -370,7 +356,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu backupId, volume.getId(), backup.getHostId(), - backup.getNbdPort(), + socket, ImageTransferVO.Phase.transferring, ImageTransfer.Direction.download, backup.getAccountId(), @@ -398,18 +384,17 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { return hosts.get(0); } - private void startNBDServer(String transferId, String direction, Host host, String exportName, String volumePath, int nbdPort) { + private void startNBDServer(String transferId, String direction, Long hostId, String exportName, String volumePath) { StartNBDServerAnswer nbdServerAnswer; StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, - host.getPrivateIpAddress(), exportName, volumePath, - nbdPort, + transferId, direction ); try { - nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(host.getId(), nbdServerCmd); + nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(hostId, nbdServerCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -451,19 +436,18 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer transferCmd = new CreateImageTransferCommand( transferId, - host.getPrivateIpAddress(), direction, + transferId, volumePath); } else { - int nbdPort = allocateNbdPort(); - startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort); + startNBDServer(transferId, direction, host.getId(), volume.getUuid(), volumePath); imageTransfer = new ImageTransferVO( transferId, null, volume.getId(), host.getId(), - nbdPort, + transferId, ImageTransferVO.Phase.transferring, ImageTransfer.Direction.upload, volume.getAccountId(), @@ -472,16 +456,17 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer transferCmd = new CreateImageTransferCommand( transferId, - host.getPrivateIpAddress(), direction, volume.getUuid(), - nbdPort, + transferId, null); } - - - EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId()); - CreateImageTransferAnswer transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); + CreateImageTransferAnswer transferAnswer; + try { + transferAnswer = (CreateImageTransferAnswer) agentManager.send(imageTransfer.getHostId(), transferCmd); + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent", e); + } if (!transferAnswer.getResult()) { if (!backend.equals(ImageTransfer.Backend.file)) { @@ -554,32 +539,27 @@ public boolean cancelImageTransfer(long imageTransferId) { private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); - int nbdPort = imageTransfer.getNbdPort(); - String direction = imageTransfer.getDirection().toString(); - FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); + FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId); BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); + Answer answer; try { - Answer answer; if (dummyOffering) { answer = new Answer(finalizeCmd, true, "Image transfer finalized."); - } else if (DATAPLANE_PROXY_MODE) { - EndPoint ssvm = _epSelector.findSsvm(backup.getZoneId()); - answer = ssvm.sendMessage(finalizeCmd); } else { answer = agentManager.send(backup.getHostId(), finalizeCmd); } - if (!answer.getResult()) { - throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); - } - } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } + if (!answer.getResult()) { + throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); + } + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); if (vm.getState() == State.Stopped) { boolean stopNbdServerResult = stopNbdServer(imageTransfer); @@ -591,9 +571,8 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { private boolean stopNbdServer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); - int nbdPort = imageTransfer.getNbdPort(); String direction = imageTransfer.getDirection().toString(); - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); + StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction); Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); @@ -606,17 +585,19 @@ private boolean stopNbdServer(ImageTransferVO imageTransfer) { private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); - int nbdPort = imageTransfer.getNbdPort(); - String direction = imageTransfer.getDirection().toString(); boolean stopNbdServerResult = stopNbdServer(imageTransfer); if (!stopNbdServerResult) { throw new CloudRuntimeException("Failed to stop the nbd server"); } - FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); - EndPoint ssvm = _epSelector.findSsvm(imageTransfer.getDataCenterId()); - Answer answer = ssvm.sendMessage(finalizeCmd); + FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId); + Answer answer; + try { + answer = agentManager.send(imageTransfer.getHostId(), finalizeCmd); + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent", e); + } if (!answer.getResult()) { throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); @@ -717,19 +698,6 @@ public List> getCommands() { return cmdList; } - private int getRandomNbdPort() { - Random random = new Random(); - return NBD_PORT_RANGE_START + random.nextInt(NBD_PORT_RANGE_END - NBD_PORT_RANGE_START); - } - - private int allocateNbdPort() { - int port = getRandomNbdPort(); - while (imageTransferDao.findByNbdPort(port) != null) { - port = getRandomNbdPort(); - } - return port; - } - private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTransferVO) { ImageTransferResponse response = new ImageTransferResponse(); response.setId(imageTransferVO.getUuid()); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 2358bdcc8324..db95a58f222f 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -54,10 +54,6 @@ import javax.naming.ConfigurationException; -import org.apache.cloudstack.backup.CreateImageTransferAnswer; -import org.apache.cloudstack.backup.CreateImageTransferCommand; -import org.apache.cloudstack.backup.FinalizeImageTransferCommand; -import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.framework.security.keystore.KeystoreManager; import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -342,10 +338,6 @@ public Answer executeRequest(Command cmd) { return execute((ListDataStoreObjectsCommand)cmd); } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { return execute((QuerySnapshotZoneCopyCommand)cmd); - } else if (cmd instanceof CreateImageTransferCommand) { - return execute((CreateImageTransferCommand)cmd); - } else if (cmd instanceof FinalizeImageTransferCommand) { - return execute((FinalizeImageTransferCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -3716,212 +3708,4 @@ protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { return new QuerySnapshotZoneCopyAnswer(cmd, files); } - private void resetService(String unitName) { - Script resetScript = new Script("/bin/bash", logger); - resetScript.add("-c"); - resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); - resetScript.execute(); - } - - private boolean stopImageServer() { - String unitName = "cloudstack-image-server"; - final int imageServerPort = 54323; - - Script checkScript = new Script("/bin/bash", logger); - checkScript.add("-c"); - checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); - String checkResult = checkScript.execute(); - if (checkResult != null) { - logger.info(String.format("Image server not running, resetting failed state")); - resetService(unitName); - // Still try to remove firewall rule in case it exists - if (_inSystemVM) { - removeFirewallRule(imageServerPort); - } - return true; - } - - Script stopScript = new Script("/bin/bash", logger); - stopScript.add("-c"); - stopScript.add(String.format("systemctl stop %s", unitName)); - stopScript.execute(); - resetService(unitName); - logger.info(String.format("Image server %s stopped", unitName)); - - // Close firewall port for image server - if (_inSystemVM) { - removeFirewallRule(imageServerPort); - } - - return true; - } - - private void removeFirewallRule(int port) { - String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); - Script removeScript = new Script("/bin/bash", logger); - removeScript.add("-c"); - removeScript.add(String.format("iptables -D INPUT %s || true", rule)); - String result = removeScript.execute(); - if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { - logger.debug(String.format("Firewall rule removal result for port %d: %s", port, result)); - } else { - logger.info(String.format("Firewall rule removed for port %d (or did not exist)", port)); - } - } - - private boolean startImageServerIfNotRunning(int imageServerPort) { - final String imageServerScript = "/opt/cloud/bin/image_server.py"; - String unitName = "cloudstack-image-server"; - - Script checkScript = new Script("/bin/bash", logger); - checkScript.add("-c"); - checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); - String checkResult = checkScript.execute(); - if (checkResult == null) { - return true; - } - - String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port %d", - unitName, imageServerScript, imageServerPort); - - Script startScript = new Script("/bin/bash", logger); - startScript.add("-c"); - startScript.add(systemdRunCmd); - String startResult = startScript.execute(); - - if (startResult != null) { - logger.error(String.format("Failed to start the Image serer: %s", startResult)); - return false; - } - - // Wait with timeout until the service is up - int maxWaitSeconds = 10; - int pollIntervalMs = 1000; - int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; - boolean serviceActive = false; - - for (int attempt = 0; attempt < maxAttempts; attempt++) { - Script verifyScript = new Script("/bin/bash", logger); - verifyScript.add("-c"); - verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); - String verifyResult = verifyScript.execute(); - if (verifyResult == null) { - serviceActive = true; - logger.info(String.format("Image server is now active (attempt %d)", attempt + 1)); - break; - } - try { - Thread.sleep(pollIntervalMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - - if (!serviceActive) { - logger.error(String.format("Image server failed to start within %d seconds", maxWaitSeconds)); - return false; - } - - // Open firewall port for image server - if (_inSystemVM) { - String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); - IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, false, rule, - String.format("Error in opening up image server port %d", imageServerPort)); - } - - return true; - } - - protected Answer execute(CreateImageTransferCommand cmd) { - if (!_inSystemVM) { - return new CreateImageTransferAnswer(cmd, true, "Not running inside SSVM; skipping image transfer setup."); - } - - final String transferId = cmd.getTransferId(); - final String hostIp = cmd.getHostIpAddress(); - final ImageTransfer.Backend backend = cmd.getBackend(); - - if (StringUtils.isBlank(transferId)) { - return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); - } - if (StringUtils.isBlank(hostIp)) { - return new CreateImageTransferAnswer(cmd, false, "hostIpAddress is empty."); - } - - final Map payload = new HashMap<>(); - payload.put("backend", backend.toString()); - - if (backend == ImageTransfer.Backend.file) { - final String filePath = cmd.getFile(); - if (StringUtils.isBlank(filePath)) { - return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); - } - payload.put("file", filePath); - } else { - final String exportName = cmd.getExportName(); - final int nbdPort = cmd.getNbdPort(); - if (StringUtils.isBlank(exportName)) { - return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); - } - if (nbdPort <= 0) { - return new CreateImageTransferAnswer(cmd, false, "Invalid nbdPort: " + nbdPort); - } - payload.put("host", hostIp); - payload.put("port", nbdPort); - payload.put("export", exportName); - String checkpointId = cmd.getCheckpointId(); - if (checkpointId != null) { - payload.put("export_bitmap", exportName + "-" + checkpointId.substring(0, 4)); - } - } - - try { - final String json = new GsonBuilder().create().toJson(payload); - File dir = new File("/tmp/imagetransfer"); - if (!dir.exists()) { - dir.mkdirs(); - } - final File transferFile = new File("/tmp/imagetransfer", transferId); - FileUtils.writeStringToFile(transferFile, json, "UTF-8"); - - } catch (IOException e) { - logger.warn("Failed to prepare image transfer on SSVM", e); - return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on SSVM: " + e.getMessage()); - } - - final int imageServerPort = 54323; - startImageServerIfNotRunning(imageServerPort); - - final String transferUrl = String.format("http://%s:%d/images/%s", _publicIp, imageServerPort, transferId); - return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on SSVM.", transferId, transferUrl); - } - - protected Answer execute(FinalizeImageTransferCommand cmd) { - if (!_inSystemVM) { - return new Answer(cmd, true, "Not running inside SSVM; skipping image transfer finalization."); - } - - final String transferId = cmd.getTransferId(); - if (StringUtils.isBlank(transferId)) { - return new Answer(cmd, false, "transferId is empty."); - } - - final File transferFile = new File("/tmp/imagetransfer", transferId); - if (transferFile.exists() && !transferFile.delete()) { - return new Answer(cmd, false, "Failed to delete transfer config file: " + transferFile.getAbsolutePath()); - } - - try (Stream stream = Files.list(Paths.get("/tmp/imagetransfer"))) { - if (!stream.findAny().isPresent()) { - stopImageServer(); - } - } catch (IOException e) { - logger.warn("Failed to list /tmp/imagetransfer", e); - } - - return new Answer(cmd, true, "Image transfer finalized."); - } - } From 0ff4dc55132ac2742ac6a477b68a2b5e2974021b Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sat, 21 Feb 2026 07:59:02 +0530 Subject: [PATCH 043/129] remove image server from systmvm --- systemvm/debian/opt/cloud/bin/image_server.py | 1529 ----------------- 1 file changed, 1529 deletions(-) delete mode 100644 systemvm/debian/opt/cloud/bin/image_server.py diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py deleted file mode 100644 index a176513698c5..000000000000 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ /dev/null @@ -1,1529 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -POC "imageio-like" HTTP server backed by NBD over TCP or a local file. - -Supports two backends (see config payload): -- nbd: proxy to an NBD server (port, export, export_bitmap); supports range reads/writes, extents, zero, flush. -- file: read/write a local qcow2 (or raw) file path; full PUT only (no range writes), GET with optional ranges, flush. - -How to run ----------- -- Install dependency: - dnf install python3-libnbd - or - apt install python3-libnbd - -- Run server: - createImageTransfer will start the server as a systemd service 'cloudstack-image-server' - -Example curl commands --------------------- -- OPTIONS: - curl -i -X OPTIONS http://127.0.0.1:54323/images/demo - -- GET full image: - curl -v http://127.0.0.1:54323/images/demo -o demo.img - -- GET a byte range: - curl -v -H "Range: bytes=0-1048575" http://127.0.0.1:54323/images/demo -o first_1MiB.bin - -- PUT full image (Content-Length must equal export size exactly): - curl -v -T demo.img http://127.0.0.1:54323/images/demo - -- GET extents (zero/hole extents from NBD base:allocation): - curl -s http://127.0.0.1:54323/images/demo/extents | jq . - -- GET extents with dirty and zero (requires export_bitmap in config): - curl -s "http://127.0.0.1:54323/images/demo/extents?context=dirty" | jq . - -- POST flush: - curl -s -X POST http://127.0.0.1:54323/images/demo/flush | jq . - -- PATCH zero (zero a byte range; application/json body): - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "offset": 4096, "size": 8192}' \ - http://127.0.0.1:54323/images/demo - - Zero at offset 1 GiB, 4096 bytes, no flush: - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "offset": 1073741824, "size": 4096}' \ - http://127.0.0.1:54323/images/demo - - Zero entire disk and flush: - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "size": 107374182400, "flush": true}' \ - http://127.0.0.1:54323/images/demo - -- PATCH flush (flush data to storage; operates on entire image): - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "flush"}' \ - http://127.0.0.1:54323/images/demo - -- PATCH range (write binary body at byte range; Range + Content-Length required): - curl -v -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin \ - http://127.0.0.1:54323/images/demo -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import socket -import threading -import time -from http import HTTPStatus -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Dict, List, Optional, Tuple -from urllib.parse import parse_qs -import nbd - -CHUNK_SIZE = 256 * 1024 # 256 KiB - -# NBD base:allocation flags (hole=1, zero=2; hole|zero=3) -_NBD_STATE_HOLE = 1 -_NBD_STATE_ZERO = 2 -# NBD qemu:dirty-bitmap flags (dirty=1) -_NBD_STATE_DIRTY = 1 - -# Concurrency limits across ALL images. -MAX_PARALLEL_READS = 8 -MAX_PARALLEL_WRITES = 1 - -_READ_SEM = threading.Semaphore(MAX_PARALLEL_READS) -_WRITE_SEM = threading.Semaphore(MAX_PARALLEL_WRITES) - -# In-memory per-image lock: single lock gates both read and write. -_IMAGE_LOCKS: Dict[str, threading.Lock] = {} -_IMAGE_LOCKS_GUARD = threading.Lock() - - -# Dynamic image_id(transferId) -> backend mapping: -# CloudStack writes a JSON file at /tmp/imagetransfer/ with: -# - NBD backend: {"backend": "nbd", "host": "...", "port": 10809, "export": "vda", "export_bitmap": "..."} -# - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} -# -# This server reads that file on-demand. -_CFG_DIR = "/tmp/imagetransfer" -_CFG_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {} -_CFG_CACHE_GUARD = threading.Lock() - - -def _json_bytes(obj: Any) -> bytes: - return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - - -def _merge_dirty_zero_extents( - allocation_extents: List[Tuple[int, int, bool]], - dirty_extents: List[Tuple[int, int, bool]], - size: int, -) -> List[Dict[str, Any]]: - """ - Merge allocation (start, length, zero) and dirty (start, length, dirty) extents - into a single list of {start, length, dirty, zero} with unified boundaries. - """ - boundaries: set[int] = {0, size} - for start, length, _ in allocation_extents: - boundaries.add(start) - boundaries.add(start + length) - for start, length, _ in dirty_extents: - boundaries.add(start) - boundaries.add(start + length) - sorted_boundaries = sorted(boundaries) - - def lookup( - extents: List[Tuple[int, int, bool]], offset: int, default: bool - ) -> bool: - for start, length, flag in extents: - if start <= offset < start + length: - return flag - return default - - result: List[Dict[str, Any]] = [] - for i in range(len(sorted_boundaries) - 1): - a, b = sorted_boundaries[i], sorted_boundaries[i + 1] - if a >= b: - continue - result.append( - { - "start": a, - "length": b - a, - "dirty": lookup(dirty_extents, a, False), - "zero": lookup(allocation_extents, a, False), - } - ) - return result - - -def _is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: - """True if extents is the single-extent fallback (dirty=false, zero=false).""" - return ( - len(extents) == 1 - and extents[0].get("dirty") is False - and extents[0].get("zero") is False - ) - - -def _get_image_lock(image_id: str) -> threading.Lock: - with _IMAGE_LOCKS_GUARD: - lock = _IMAGE_LOCKS.get(image_id) - if lock is None: - lock = threading.Lock() - _IMAGE_LOCKS[image_id] = lock - return lock - - -def _now_s() -> float: - return time.monotonic() - - -def _safe_transfer_id(image_id: str) -> Optional[str]: - """ - Only allow a single filename component to avoid path traversal. - We intentionally keep validation simple: reject anything containing '/' or '\\'. - """ - if not image_id: - return None - if image_id != os.path.basename(image_id): - return None - if "/" in image_id or "\\" in image_id: - return None - if image_id in (".", ".."): - return None - return image_id - - -def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: - safe_id = _safe_transfer_id(image_id) - if safe_id is None: - return None - - cfg_path = os.path.join(_CFG_DIR, safe_id) - try: - st = os.stat(cfg_path) - except FileNotFoundError: - return None - except OSError as e: - logging.error("cfg stat failed image_id=%s err=%r", image_id, e) - return None - - with _CFG_CACHE_GUARD: - cached = _CFG_CACHE.get(safe_id) - if cached is not None: - cached_mtime, cached_cfg = cached - # Use cached config if the file hasn't changed. - if float(st.st_mtime) == float(cached_mtime): - return cached_cfg - - try: - with open(cfg_path, "rb") as f: - raw = f.read(4096) - except OSError as e: - logging.error("cfg read failed image_id=%s err=%r", image_id, e) - return None - - try: - obj = json.loads(raw.decode("utf-8")) - except Exception as e: - logging.error("cfg parse failed image_id=%s err=%r", image_id, e) - return None - - if not isinstance(obj, dict): - logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) - return None - - backend = obj.get("backend") - if backend is None: - backend = "nbd" - if not isinstance(backend, str): - logging.error("cfg invalid backend type image_id=%s", image_id) - return None - backend = backend.lower() - if backend not in ("nbd", "file"): - logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) - return None - - if backend == "file": - file_path = obj.get("file") - if not isinstance(file_path, str) or not file_path.strip(): - logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) - return None - cfg = {"backend": "file", "file": file_path.strip()} - else: - host = obj.get("host") - port = obj.get("port") - export = obj.get("export") - export_bitmap = obj.get("export_bitmap") - if not isinstance(host, str) or not host: - logging.error("cfg missing/invalid host image_id=%s", image_id) - return None - try: - port_i = int(port) - except Exception: - logging.error("cfg missing/invalid port image_id=%s", image_id) - return None - if port_i <= 0 or port_i > 65535: - logging.error("cfg out-of-range port image_id=%s port=%r", image_id, port) - return None - if export is not None and (not isinstance(export, str) or not export): - logging.error("cfg missing/invalid export image_id=%s", image_id) - return None - cfg = { - "backend": "nbd", - "host": host, - "port": port_i, - "export": export, - "export_bitmap": export_bitmap, - } - - with _CFG_CACHE_GUARD: - _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) - return cfg - - -class _NbdConn: - """ - Small helper to connect to NBD using an already-open TCP socket. - Opens a fresh handle per request, per POC requirements. - """ - - def __init__( - self, - host: str, - port: int, - export: Optional[str], - need_block_status: bool = False, - extra_meta_contexts: Optional[List[str]] = None, - ): - self._sock = socket.create_connection((host, port)) - self._nbd = nbd.NBD() - - # Select export name if supported/needed. - if export and hasattr(self._nbd, "set_export_name"): - self._nbd.set_export_name(export) - - # Request meta contexts before connect (for block status / dirty bitmap). - if need_block_status and hasattr(self._nbd, "add_meta_context"): - for ctx in ["base:allocation"] + (extra_meta_contexts or []): - try: - self._nbd.add_meta_context(ctx) - except Exception as e: - logging.warning("add_meta_context %r failed: %r", ctx, e) - - self._connect_existing_socket(self._sock) - - def _connect_existing_socket(self, sock: socket.socket) -> None: - # Requirement: attach libnbd to an existing socket / FD (no qemu-nbd). - # libnbd python API varies slightly by version, so try common options. - last_err: Optional[BaseException] = None - if hasattr(self._nbd, "connect_socket"): - try: - self._nbd.connect_socket(sock) - return - except Exception as e: # pragma: no cover (depends on binding) - last_err = e - try: - self._nbd.connect_socket(sock.fileno()) - return - except Exception as e2: # pragma: no cover - last_err = e2 - if hasattr(self._nbd, "connect_fd"): - try: - self._nbd.connect_fd(sock.fileno()) - return - except Exception as e: # pragma: no cover - last_err = e - raise RuntimeError( - "Unable to connect libnbd using existing socket/fd; " - f"binding missing connect_socket/connect_fd or call failed: {last_err!r}" - ) - - def size(self) -> int: - return int(self._nbd.get_size()) - - def get_capabilities(self) -> Dict[str, bool]: - """ - Query NBD export capabilities (read_only, can_flush, can_zero) from the - server handshake. Returns dict with keys read_only, can_flush, can_zero. - Uses getattr for binding name variations (is_read_only/get_read_only, etc.). - """ - out: Dict[str, bool] = { - "read_only": True, - "can_flush": False, - "can_zero": False, - } - for name, keys in [ - ("read_only", ("is_read_only", "get_read_only")), - ("can_flush", ("can_flush", "get_can_flush")), - ("can_zero", ("can_zero", "get_can_zero")), - ]: - for attr in keys: - if hasattr(self._nbd, attr): - try: - val = getattr(self._nbd, attr)() - out[name] = bool(val) - except Exception: - pass - break - return out - - def pread(self, length: int, offset: int) -> bytes: - # Expected signature: pread(length, offset) - try: - return self._nbd.pread(length, offset) - except TypeError: # pragma: no cover (binding differences) - return self._nbd.pread(offset, length) - - def pwrite(self, buf: bytes, offset: int) -> None: - # Expected signature: pwrite(buf, offset) - try: - self._nbd.pwrite(buf, offset) - except TypeError: # pragma: no cover (binding differences) - self._nbd.pwrite(offset, buf) - - def pzero(self, offset: int, size: int) -> None: - """ - Zero a byte range. Uses NBD WRITE_ZEROES when available (efficient/punch hole), - otherwise falls back to writing zero bytes via pwrite. - """ - if size <= 0: - return - # Try libnbd pwrite_zeros / zero; argument order varies by binding. - for name in ("pwrite_zeros", "zero"): - if not hasattr(self._nbd, name): - continue - fn = getattr(self._nbd, name) - try: - fn(size, offset) - return - except TypeError: - try: - fn(offset, size) - return - except TypeError: - pass - # Fallback: write zeros in chunks. - remaining = size - pos = offset - zero_buf = b"\x00" * min(CHUNK_SIZE, size) - while remaining > 0: - chunk = min(len(zero_buf), remaining) - self.pwrite(zero_buf[:chunk], pos) - pos += chunk - remaining -= chunk - - def flush(self) -> None: - if hasattr(self._nbd, "flush"): - self._nbd.flush() - return - if hasattr(self._nbd, "fsync"): - self._nbd.fsync() - return - raise RuntimeError("libnbd binding has no flush/fsync method") - - def get_zero_extents(self) -> List[Dict[str, Any]]: - """ - Query NBD block status (base:allocation) and return extents that are - hole or zero in imageio format: [{"start": ..., "length": ..., "zero": true}, ...]. - Returns [] if block status is not supported; fallback to one full-image - zero extent when we have size but block status fails. - """ - size = self.size() - if size == 0: - return [] - - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - logging.error("get_zero_extents: no block_status/block_status_64") - return self._fallback_zero_extent(size) - if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( - "base:allocation" - ): - logging.error( - "get_zero_extents: server did not negotiate base:allocation" - ) - return self._fallback_zero_extent(size) - - zero_extents: List[Dict[str, Any]] = [] - chunk = min(size, 64 * 1024 * 1024) # 64 MiB - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - # Binding typically passes (metacontext, offset, entries[, nr_entries][, error]). - metacontext = None - off = 0 - entries = None - if len(args) >= 3: - metacontext, off, entries = args[0], args[1], args[2] - else: - for a in args: - if isinstance(a, str): - metacontext = a - elif isinstance(a, int): - off = a - elif a is not None and hasattr(a, "__iter__"): - entries = a - if metacontext != "base:allocation" or entries is None: - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - if (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0: - zero_extents.append( - {"start": current, "length": length, "zero": True} - ) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return self._fallback_zero_extent(size) - - try: - while offset < size: - count = min(chunk, size - offset) - # Try (count, offset, callback) then (offset, count, callback) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.error("get_zero_extents block_status failed: %r", e) - return self._fallback_zero_extent(size) - if not zero_extents: - return self._fallback_zero_extent(size) - return zero_extents - - def _fallback_zero_extent(self, size: int) -> List[Dict[str, Any]]: - """Return one zero extent covering the whole image when block status unavailable.""" - return [{"start": 0, "length": size, "zero": True}] - - def get_allocation_extents(self) -> List[Dict[str, Any]]: - """ - Query base:allocation and return all extents (allocated and hole/zero) - as [{"start": ..., "length": ..., "zero": bool}, ...]. - Fallback when block status unavailable: one extent with zero=False. - """ - size = self.size() - if size == 0: - return [] - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - return [{"start": 0, "length": size, "zero": False}] - if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( - "base:allocation" - ): - return [{"start": 0, "length": size, "zero": False}] - - allocation_extents: List[Dict[str, Any]] = [] - chunk = min(size, 64 * 1024 * 1024) - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - if len(args) < 3: - return 0 - metacontext, off, entries = args[0], args[1], args[2] - if metacontext != "base:allocation" or entries is None: - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 - allocation_extents.append( - {"start": current, "length": length, "zero": zero} - ) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return [{"start": 0, "length": size, "zero": False}] - try: - while offset < size: - count = min(chunk, size - offset) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.warning("get_allocation_extents block_status failed: %r", e) - return [{"start": 0, "length": size, "zero": False}] - if not allocation_extents: - return [{"start": 0, "length": size, "zero": False}] - return allocation_extents - - def get_extents_dirty_and_zero( - self, dirty_bitmap_context: str - ) -> List[Dict[str, Any]]: - """ - Query block status for base:allocation and qemu:dirty-bitmap:, - merge boundaries, and return extents with dirty and zero flags. - Format: [{"start": ..., "length": ..., "dirty": bool, "zero": bool}, ...]. - """ - size = self.size() - if size == 0: - return [] - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - return self._fallback_dirty_zero_extents(size) - if hasattr(self._nbd, "can_meta_context"): - if not self._nbd.can_meta_context("base:allocation"): - return self._fallback_dirty_zero_extents(size) - if not self._nbd.can_meta_context(dirty_bitmap_context): - logging.warning( - "dirty bitmap context %r not negotiated", dirty_bitmap_context - ) - return self._fallback_dirty_zero_extents(size) - - allocation_extents: List[Tuple[int, int, bool]] = [] # (start, length, zero) - dirty_extents: List[Tuple[int, int, bool]] = [] # (start, length, dirty) - chunk = min(size, 64 * 1024 * 1024) - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - if len(args) < 3: - return 0 - metacontext, off, entries = args[0], args[1], args[2] - if entries is None or not hasattr(entries, "__iter__"): - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - if metacontext == "base:allocation": - zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 - allocation_extents.append((current, length, zero)) - elif metacontext == dirty_bitmap_context: - dirty = (flags & _NBD_STATE_DIRTY) != 0 - dirty_extents.append((current, length, dirty)) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return self._fallback_dirty_zero_extents(size) - try: - while offset < size: - count = min(chunk, size - offset) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.warning("get_extents_dirty_and_zero block_status failed: %r", e) - return self._fallback_dirty_zero_extents(size) - return _merge_dirty_zero_extents(allocation_extents, dirty_extents, size) - - def _fallback_dirty_zero_extents(self, size: int) -> List[Dict[str, Any]]: - """One extent: whole image, dirty=false, zero=false when bitmap unavailable.""" - return [{"start": 0, "length": size, "dirty": False, "zero": False}] - - def close(self) -> None: - # Best-effort; bindings may differ. - try: - if hasattr(self._nbd, "shutdown"): - self._nbd.shutdown() - except Exception: - pass - try: - if hasattr(self._nbd, "close"): - self._nbd.close() - except Exception: - pass - try: - self._sock.close() - except Exception: - pass - - def __enter__(self) -> "_NbdConn": - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - -class Handler(BaseHTTPRequestHandler): - server_version = "imageio-poc/0.1" - - # Keep BaseHTTPRequestHandler from printing noisy default logs - def log_message(self, fmt: str, *args: Any) -> None: - logging.info("%s - - %s", self.address_string(), fmt % args) - - def _send_imageio_headers( - self, allowed_methods: Optional[str] = None - ) -> None: - # Include these headers for compatibility with the imageio contract. - if allowed_methods is None: - allowed_methods = "GET, PUT, OPTIONS" - self.send_header("Access-Control-Allow-Methods", allowed_methods) - self.send_header("Accept-Ranges", "bytes") - - def _send_json( - self, - status: int, - obj: Any, - allowed_methods: Optional[str] = None, - ) -> None: - body = _json_bytes(obj) - self.send_response(status) - self._send_imageio_headers(allowed_methods) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - try: - self.wfile.write(body) - except BrokenPipeError: - pass - - def _send_error_json(self, status: int, message: str) -> None: - self._send_json(status, {"error": message}) - - def _send_range_not_satisfiable(self, size: int) -> None: - # RFC 7233: reply with Content-Range: bytes */ - self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) - self._send_imageio_headers() - self.send_header("Content-Type", "application/json") - self.send_header("Content-Range", f"bytes */{size}") - body = _json_bytes({"error": "range not satisfiable"}) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - try: - self.wfile.write(body) - except BrokenPipeError: - pass - - def _parse_single_range(self, range_header: str, size: int) -> Tuple[int, int]: - """ - Parse a single HTTP byte range (RFC 7233) and return (start, end_inclusive). - - Supported: - - Range: bytes=START-END - - Range: bytes=START- - - Range: bytes=-SUFFIX - - Raises ValueError for invalid headers. Caller handles 416 vs 400. - """ - if size < 0: - raise ValueError("invalid size") - if not range_header: - raise ValueError("empty Range") - if "," in range_header: - raise ValueError("multiple ranges not supported") - - prefix = "bytes=" - if not range_header.startswith(prefix): - raise ValueError("only bytes ranges supported") - spec = range_header[len(prefix) :].strip() - if "-" not in spec: - raise ValueError("invalid bytes range") - - left, right = spec.split("-", 1) - left = left.strip() - right = right.strip() - - if left == "": - # Suffix range: last N bytes. - if right == "": - raise ValueError("invalid suffix range") - try: - suffix_len = int(right, 10) - except ValueError as e: - raise ValueError("invalid suffix length") from e - if suffix_len <= 0: - raise ValueError("invalid suffix length") - if size == 0: - # Nothing to serve - raise ValueError("unsatisfiable") - if suffix_len >= size: - return 0, size - 1 - return size - suffix_len, size - 1 - - # START is present - try: - start = int(left, 10) - except ValueError as e: - raise ValueError("invalid range start") from e - if start < 0: - raise ValueError("invalid range start") - if start >= size: - raise ValueError("unsatisfiable") - - if right == "": - # START- - return start, size - 1 - - try: - end = int(right, 10) - except ValueError as e: - raise ValueError("invalid range end") from e - if end < start: - raise ValueError("unsatisfiable") - if end >= size: - end = size - 1 - return start, end - - def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: - # Returns (image_id, tail) where tail is: - # None => /images/{id} - # "extents" => /images/{id}/extents - # "flush" => /images/{id}/flush - path = self.path.split("?", 1)[0] - parts = [p for p in path.split("/") if p] - if len(parts) < 2 or parts[0] != "images": - return None, None - image_id = parts[1] - tail = parts[2] if len(parts) >= 3 else None - if len(parts) > 3: - return None, None - return image_id, tail - - def _parse_query(self) -> Dict[str, List[str]]: - """Parse query string from self.path into a dict of name -> list of values.""" - if "?" not in self.path: - return {} - query = self.path.split("?", 1)[1] - return parse_qs(query, keep_blank_values=True) - - def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: - return _load_image_cfg(image_id) - - def _is_file_backend(self, cfg: Dict[str, Any]) -> bool: - return cfg.get("backend") == "file" - - def do_OPTIONS(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - if self._is_file_backend(cfg): - # File backend: full PUT only, no range writes; GET with ranges allowed; flush supported. - allowed_methods = "GET, PUT, POST, OPTIONS" - features = ["flush"] - max_writers = MAX_PARALLEL_WRITES - response = { - "unix_socket": None, - "features": features, - "max_readers": MAX_PARALLEL_READS, - "max_writers": max_writers, - } - self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - return - # Query NBD backend for capabilities (like nbdinfo); fall back to config. - read_only = True - can_flush = False - can_zero = False - try: - with _NbdConn( - cfg["host"], - int(cfg["port"]), - cfg.get("export"), - ) as conn: - caps = conn.get_capabilities() - read_only = caps["read_only"] - can_flush = caps["can_flush"] - can_zero = caps["can_zero"] - except Exception as e: - logging.warning("OPTIONS: could not query NBD capabilities: %r", e) - read_only = bool(cfg.get("read_only")) - if not read_only: - can_flush = True - can_zero = True - # Report options for this image from NBD: read-only => no PUT; only advertise supported features. - if read_only: - allowed_methods = "GET, OPTIONS" - features = ["extents"] - max_writers = 0 - else: - # PATCH: JSON (zero/flush) and Range+binary (write byte range). - allowed_methods = "GET, PUT, PATCH, OPTIONS" - features = ["extents"] - if can_zero: - features.append("zero") - if can_flush: - features.append("flush") - max_writers = MAX_PARALLEL_WRITES if not read_only else 0 - response = { - "unix_socket": None, # Not used in this implementation - "features": features, - "max_readers": MAX_PARALLEL_READS, - "max_writers": max_writers, - } - self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - - def do_GET(self) -> None: - image_id, tail = self._parse_route() - if image_id is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - if tail == "extents": - if self._is_file_backend(cfg): - self._send_error_json( - HTTPStatus.BAD_REQUEST, "extents not supported for file backend" - ) - return - query = self._parse_query() - context = (query.get("context") or [None])[0] - self._handle_get_extents(image_id, cfg, context=context) - return - if tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - range_header = self.headers.get("Range") - self._handle_get_image(image_id, cfg, range_header) - - def do_PUT(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - if self.headers.get("Range") is not None or self.headers.get("Content-Range") is not None: - self._send_error_json( - HTTPStatus.BAD_REQUEST, "Range/Content-Range not supported; full writes only" - ) - return - - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - - self._handle_put_image(image_id, cfg, content_length) - - def do_POST(self) -> None: - image_id, tail = self._parse_route() - if image_id is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - if tail == "flush": - self._handle_post_flush(image_id, cfg) - return - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - - def do_PATCH(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - if self._is_file_backend(cfg): - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "range writes and PATCH not supported for file backend; use PUT for full upload", - ) - return - - content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() - range_header = self.headers.get("Range") - - # Binary PATCH: Range + body writes bytes at that range (e.g. curl -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin). - if range_header is not None and content_type != "application/json": - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") - return - self._handle_patch_range(image_id, cfg, range_header, content_length) - return - - # JSON PATCH: application/json with op (zero, flush). - if content_type != "application/json": - self._send_error_json( - HTTPStatus.UNSUPPORTED_MEDIA_TYPE, - "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", - ) - return - - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length <= 0 or content_length > 64 * 1024: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - - body = self.rfile.read(content_length) - if len(body) != content_length: - self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") - return - - try: - payload = json.loads(body.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError) as e: - self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") - return - - if not isinstance(payload, dict): - self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") - return - - op = payload.get("op") - if op == "flush": - # Flush entire image; offset and size are ignored (per spec). - self._handle_post_flush(image_id, cfg) - return - if op != "zero": - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "unsupported op; only \"zero\" and \"flush\" are supported", - ) - return - - try: - size = int(payload.get("size")) - except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") - return - if size <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") - return - - offset = payload.get("offset") - if offset is None: - offset = 0 - else: - try: - offset = int(offset) - except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") - return - if offset < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") - return - - flush = bool(payload.get("flush", False)) - - self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) - - def _handle_get_image( - self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] - ) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _READ_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") - return - - start = _now_s() - bytes_sent = 0 - try: - logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") - if self._is_file_backend(cfg): - file_path = cfg["file"] - try: - size = os.path.getsize(file_path) - except OSError as e: - logging.error("GET file size error image_id=%s path=%s err=%r", image_id, file_path, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "failed to access file") - return - start_off = 0 - end_off_incl = size - 1 if size > 0 else -1 - status = HTTPStatus.OK - content_length = size - if range_header is not None: - try: - start_off, end_off_incl = self._parse_single_range(range_header, size) - except ValueError as e: - if str(e) == "unsatisfiable": - self._send_range_not_satisfiable(size) - return - if "unsatisfiable" in str(e): - self._send_range_not_satisfiable(size) - return - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") - return - status = HTTPStatus.PARTIAL_CONTENT - content_length = (end_off_incl - start_off) + 1 - - self.send_response(status) - self._send_imageio_headers() - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(content_length)) - if status == HTTPStatus.PARTIAL_CONTENT: - self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") - self.end_headers() - - offset = start_off - end_excl = end_off_incl + 1 - with open(file_path, "rb") as f: - f.seek(offset) - while offset < end_excl: - to_read = min(CHUNK_SIZE, end_excl - offset) - data = f.read(to_read) - if not data: - break - try: - self.wfile.write(data) - except BrokenPipeError: - logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) - break - offset += len(data) - bytes_sent += len(data) - else: - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - size = conn.size() - - start_off = 0 - end_off_incl = size - 1 if size > 0 else -1 - status = HTTPStatus.OK - content_length = size - if range_header is not None: - try: - start_off, end_off_incl = self._parse_single_range(range_header, size) - except ValueError as e: - if str(e) == "unsatisfiable": - self._send_range_not_satisfiable(size) - return - if "unsatisfiable" in str(e): - self._send_range_not_satisfiable(size) - return - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") - return - status = HTTPStatus.PARTIAL_CONTENT - content_length = (end_off_incl - start_off) + 1 - - self.send_response(status) - self._send_imageio_headers() - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(content_length)) - if status == HTTPStatus.PARTIAL_CONTENT: - self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") - self.end_headers() - - offset = start_off - end_excl = end_off_incl + 1 - while offset < end_excl: - to_read = min(CHUNK_SIZE, end_excl - offset) - data = conn.pread(to_read, offset) - if not data: - raise RuntimeError("backend returned empty read") - try: - self.wfile.write(data) - except BrokenPipeError: - logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) - break - offset += len(data) - bytes_sent += len(data) - except Exception as e: - # If headers already sent, we can't return JSON reliably; just log. - logging.error("GET error image_id=%s err=%r", image_id, e) - try: - if not self.wfile.closed: - self.close_connection = True - except Exception: - pass - finally: - _READ_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur - ) - - def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: int) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - bytes_written = 0 - try: - logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) - if self._is_file_backend(cfg): - file_path = cfg["file"] - remaining = content_length - with open(file_path, "wb") as f: - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {bytes_written} bytes", - ) - return - f.write(chunk) - bytes_written += len(chunk) - remaining -= len(chunk) - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) - else: - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - offset = 0 - remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {offset} bytes", - ) - return - conn.pwrite(chunk, offset) - offset += len(chunk) - remaining -= len(chunk) - bytes_written += len(chunk) - - # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) - except Exception as e: - logging.error("PUT error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur - ) - - def _handle_get_extents( - self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None - ) -> None: - # context=dirty: return extents with dirty and zero from base:allocation + bitmap. - # Otherwise: return zero/hole extents from base:allocation only. - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - start = _now_s() - try: - logging.info("EXTENTS start image_id=%s context=%s", image_id, context) - if context == "dirty": - export_bitmap = cfg.get("export_bitmap") - if not export_bitmap: - # Fallback: same structure as zero extents but dirty=true for all ranges - with _NbdConn( - cfg["host"], - int(cfg["port"]), - cfg.get("export"), - need_block_status=True, - ) as conn: - allocation = conn.get_allocation_extents() - extents = [ - {"start": e["start"], "length": e["length"], "dirty": True, "zero": e["zero"]} - for e in allocation - ] - else: - dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}" - extra_contexts: List[str] = [dirty_bitmap_ctx] - with _NbdConn( - cfg["host"], - int(cfg["port"]), - cfg.get("export"), - need_block_status=True, - extra_meta_contexts=extra_contexts, - ) as conn: - extents = conn.get_extents_dirty_and_zero(dirty_bitmap_ctx) - # When bitmap not actually available, same fallback: zero structure + dirty=true - if _is_fallback_dirty_response(extents): - with _NbdConn( - cfg["host"], - int(cfg["port"]), - cfg.get("export"), - need_block_status=True, - ) as conn: - allocation = conn.get_allocation_extents() - extents = [ - { - "start": e["start"], - "length": e["length"], - "dirty": True, - "zero": e["zero"], - } - for e in allocation - ] - else: - with _NbdConn( - cfg["host"], - int(cfg["port"]), - cfg.get("export"), - need_block_status=True, - ) as conn: - extents = conn.get_zero_extents() - self._send_json(HTTPStatus.OK, extents) - except Exception as e: - logging.error("EXTENTS error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - lock.release() - dur = _now_s() - start - logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - start = _now_s() - try: - logging.info("FLUSH start image_id=%s", image_id) - if self._is_file_backend(cfg): - file_path = cfg["file"] - with open(file_path, "rb") as f: - f.flush() - os.fsync(f.fileno()) - self._send_json(HTTPStatus.OK, {"ok": True}) - else: - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - conn.flush() - self._send_json(HTTPStatus.OK, {"ok": True}) - except Exception as e: - logging.error("FLUSH error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - lock.release() - dur = _now_s() - start - logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_patch_zero( - self, - image_id: str, - cfg: Dict[str, Any], - offset: int, - size: int, - flush: bool, - ) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - try: - logging.info( - "PATCH zero start image_id=%s offset=%d size=%d flush=%s", - image_id, offset, size, flush, - ) - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - image_size = conn.size() - if offset >= image_size: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "offset must be less than image size", - ) - return - zero_size = min(size, image_size - offset) - conn.pzero(offset, zero_size) - if flush: - conn.flush() - self._send_json(HTTPStatus.OK, {"ok": True}) - except Exception as e: - logging.error("PATCH zero error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_patch_range( - self, - image_id: str, - cfg: Dict[str, Any], - range_header: str, - content_length: int, - ) -> None: - """Write request body to the image at the byte range from Range header.""" - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - bytes_written = 0 - try: - logging.info( - "PATCH range start image_id=%s range=%s content_length=%d", - image_id, range_header, content_length, - ) - with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - image_size = conn.size() - try: - start_off, end_inclusive = self._parse_single_range( - range_header, image_size - ) - except ValueError as e: - if "unsatisfiable" in str(e).lower(): - self._send_range_not_satisfiable(image_size) - else: - self._send_error_json( - HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" - ) - return - expected_len = end_inclusive - start_off + 1 - if content_length != expected_len: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"Content-Length ({content_length}) must equal range length ({expected_len})", - ) - return - offset = start_off - remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {bytes_written} bytes", - ) - return - conn.pwrite(chunk, offset) - n = len(chunk) - offset += n - remaining -= n - bytes_written += n - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) - except Exception as e: - logging.error("PATCH range error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "PATCH range end image_id=%s bytes=%d duration_s=%.3f", - image_id, bytes_written, dur, - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") - parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") - parser.add_argument("--port", type=int, default=54323, help="Port to listen on") - args = parser.parse_args() - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - ) - - addr = (args.listen, args.port) - httpd = ThreadingHTTPServer(addr, Handler) - logging.info("listening on http://%s:%d", args.listen, args.port) - logging.info("image configs are read from %s/", _CFG_DIR) - httpd.serve_forever() - - -if __name__ == "__main__": - main() From f9070985d5724250c5889306dd0ea74d30e8bd92 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 23 Feb 2026 19:09:26 +0530 Subject: [PATCH 044/129] changes Signed-off-by: Abhishek Kumar --- .../com/cloud/tags/dao/ResourceTagDao.java | 2 + .../cloud/tags/dao/ResourceTagsDaoImpl.java | 7 + .../veeam/adapter/ServerAdapter.java | 94 ++- .../veeam/api/TagsRouteHandler.java | 102 +++ .../cloudstack/veeam/api/VmsRouteHandler.java | 24 +- .../AsyncJobJoinVOToJobConverter.java | 2 +- .../ResourceTagVOToTagConverter.java | 67 ++ .../converter/UserVmJoinVOToVmConverter.java | 45 +- .../cloudstack/veeam/api/dto/BaseDto.java | 2 + .../cloudstack/veeam/api/dto/BootMenu.java | 34 - .../veeam/api/dto/HardwareInformation.java | 69 -- .../apache/cloudstack/veeam/api/dto/Host.java | 49 ++ .../cloudstack/veeam/api/dto/HostSummary.java | 57 -- .../apache/cloudstack/veeam/api/dto/Nics.java | 41 -- .../apache/cloudstack/veeam/api/dto/Os.java | 22 + .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 671 ++++++++++++++++++ .../veeam/api/dto/{Bios.java => Tag.java} | 40 +- .../apache/cloudstack/veeam/api/dto/Vm.java | 158 ++++- .../veeam/api/dto/VmInitialization.java | 34 - .../spring-veeam-control-service-context.xml | 1 + .../src/main/resources/test.xml | 618 ++++++++++++++++ .../veeam/api/dto/OvfXmlUtilTest.java | 28 + .../backup/IncrementalBackupServiceImpl.java | 4 + 23 files changed, 1892 insertions(+), 279 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/{Bios.java => Tag.java} (58%) delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java create mode 100644 plugins/integrations/veeam-control-service/src/main/resources/test.xml create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index bacb09b98793..5f2225c410f5 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -60,4 +60,6 @@ public interface ResourceTagDao extends GenericDao { void removeByResourceIdAndKey(long resourceId, ResourceObjectType resourceType, String key); List listByResourceUuid(String resourceUuid); + + List listByResourceType(ResourceObjectType resourceType); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index cc9d99e6ab16..6fb7f71b269d 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -120,4 +120,11 @@ public List listByResourceUuid(String resourceUuid) { sc.setParameters("resourceUuid", resourceUuid); return listBy(sc); } + + @Override + public List listByResourceType(ResourceObjectType resourceType) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("resourceType", resourceType); + return listBy(sc); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 781cb6b94d66..eb2f94628fd3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.veeam.adapter; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -75,6 +76,8 @@ import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.TagsRouteHandler; import org.apache.cloudstack.veeam.api.converter.AsyncJobJoinVOToJobConverter; import org.apache.cloudstack.veeam.api.converter.BackupVOToBackupConverter; import org.apache.cloudstack.veeam.api.converter.ClusterVOToClusterConverter; @@ -84,12 +87,14 @@ import org.apache.cloudstack.veeam.api.converter.NetworkVOToNetworkConverter; import org.apache.cloudstack.veeam.api.converter.NetworkVOToVnicProfileConverter; import org.apache.cloudstack.veeam.api.converter.NicVOToNicConverter; +import org.apache.cloudstack.veeam.api.converter.ResourceTagVOToTagConverter; import org.apache.cloudstack.veeam.api.converter.StoreVOToStorageDomainConverter; import org.apache.cloudstack.veeam.api.converter.UserVmJoinVOToVmConverter; import org.apache.cloudstack.veeam.api.converter.UserVmVOToCheckpointConverter; import org.apache.cloudstack.veeam.api.converter.VmSnapshotVOToSnapshotConverter; import org.apache.cloudstack.veeam.api.converter.VolumeJoinVOToDiskConverter; import org.apache.cloudstack.veeam.api.dto.Backup; +import org.apache.cloudstack.veeam.api.dto.BaseDto; import org.apache.cloudstack.veeam.api.dto.Checkpoint; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.DataCenter; @@ -100,9 +105,11 @@ import org.apache.cloudstack.veeam.api.dto.Job; import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.Nic; +import org.apache.cloudstack.veeam.api.dto.OvfXmlUtil; import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.StorageDomain; +import org.apache.cloudstack.veeam.api.dto.Tag; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.dto.VmAction; import org.apache.cloudstack.veeam.api.dto.VnicProfile; @@ -138,12 +145,15 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; +import com.cloud.server.ResourceTag; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.tags.ResourceTagVO; +import com.cloud.tags.dao.ResourceTagDao; import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.User; @@ -266,8 +276,31 @@ public class ServerAdapter extends ManagerBase { @Inject BackupDao backupDao; + @Inject + ResourceTagDao resourceTagDao; + //ToDo: check access on objects + protected static Tag getDummyTagByName(String name) { + Tag tag = new Tag(); + String id = UUID.nameUUIDFromBytes(String.format("veeam:%s", name.toLowerCase()).getBytes()).toString(); + tag.setId(id); + tag.setName(name); + tag.setDescription(String.format("Default %s tag", name.toLowerCase())); + tag.setHref(VeeamControlService.ContextPath.value() + TagsRouteHandler.BASE_ROUTE + "/" + id); + tag.setParent(ResourceTagVOToTagConverter.getRootTagRef()); + return tag; + } + + protected static Map getDummyTags() { + Map tags = new HashMap<>(); + Tag tag1 = getDummyTagByName("Automatic"); + tags.put(tag1.getId(), tag1); + Tag tag2 = getDummyTagByName("Manual"); + tags.put(tag2.getId(), tag2); + return tags; + } + protected Role createServiceAccountRole() { Role role = roleService.createRole(SERVICE_ACCOUNT_ROLE_NAME, RoleType.User, SERVICE_ACCOUNT_ROLE_NAME, false); @@ -417,20 +450,30 @@ public List listAllInstances() { return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById); } - public Vm getInstance(String uuid) { + public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, boolean allContent) { UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::listDiskAttachmentsByInstanceId, - this::listNicsByInstance); + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, + includeDisks ? this::listDiskAttachmentsByInstanceId : null, + includeNics ? this::listNicsByInstance : null, + allContent); } public Vm createInstance(Vm request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); } + OvfXmlUtil.updateFromConfiguration(request); String name = request.getName(); + if (StringUtils.isBlank(name)) { + throw new InvalidParameterValueException("Invalid name specified for the VM"); + } + String displayName = name; + if (name.endsWith("_restored")) { + name = name.replace("_restored", "-restored"); + } Long zoneId = null; Long clusterId = null; if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { @@ -446,14 +489,16 @@ public Vm createInstance(Vm request) { Integer cpu = null; try { cpu = Integer.valueOf(request.getCpu().getTopology().getSockets()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } if (cpu == null) { throw new InvalidParameterValueException("CPU topology sockets must be specified"); } Long memory = null; try { memory = Long.valueOf(request.getMemory()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } if (memory == null) { throw new InvalidParameterValueException("Memory must be specified"); } @@ -470,7 +515,7 @@ public Vm createInstance(Vm request) { Pair serviceUserAccount = createServiceAccountIfNeeded(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - return createInstance(zoneId, clusterId, name, cpu, memory, userdata, bootType, bootMode); + return createInstance(zoneId, clusterId, name, displayName, cpu, memory, userdata, bootType, bootMode); } finally { CallContext.unregister(); } @@ -491,8 +536,8 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(long zoneId, int cpu return serviceOfferingDao.findByUuid(uuid); } - protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, long memory, String userdata, - ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { + protected Vm createInstance(Long zoneId, Long clusterId, String name, String displayName, int cpu, long memory, + String userdata, ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zoneId, cpu, memory); if (serviceOffering == null) { throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); @@ -503,6 +548,9 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, l cmd.setZoneId(zoneId); cmd.setClusterId(clusterId); cmd.setName(name); + if (displayName != null) { + cmd.setDisplayName(displayName); + } cmd.setServiceOfferingId(serviceOffering.getId()); if (StringUtils.isNotEmpty(userdata)) { cmd.setUserData(Base64.getEncoder().encodeToString(userdata.getBytes(StandardCharsets.UTF_8))); @@ -526,7 +574,7 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, l vm = userVmService.finalizeCreateVirtualMachine(vm.getId()); UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::listDiskAttachmentsByInstanceId, - this::listNicsByInstance); + this::listNicsByInstance, false); } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); } @@ -534,7 +582,7 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, int cpu, l public Vm updateInstance(String uuid, Vm request) { // ToDo: what to do?! - return getInstance(uuid); + return getInstance(uuid, false, false, false); } public void deleteInstance(String uuid) { @@ -1236,4 +1284,30 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { CallContext.unregister(); } } + + public List listAllTags() { + List tags = new ArrayList<>(getDummyTags().values()); + List vmResourceTags = resourceTagDao.listByResourceType(ResourceTag.ResourceObjectType.UserVm); + if (CollectionUtils.isNotEmpty(vmResourceTags)) { + tags.addAll(ResourceTagVOToTagConverter.toTags(vmResourceTags)); + } + return tags; + } + + public Tag getTag(String uuid) { + if (BaseDto.ZERO_UUID.equals(uuid)) { + return ResourceTagVOToTagConverter.getRootTag(); + } + Tag tag = getDummyTags().get(uuid); + if (tag == null) { + ResourceTagVO resourceTagVO = resourceTagDao.findByUuid(uuid); + if (resourceTagVO != null) { + tag = ResourceTagVOToTagConverter.toTag(resourceTagVO); + } + } + if (tag == null) { + throw new InvalidParameterValueException("Tag with ID " + uuid + " not found"); + } + return tag; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java new file mode 100644 index 000000000000..e81709cb2121 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api; + +import java.io.IOException; +import java.util.List; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.RouteHandler; +import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; +import org.apache.cloudstack.veeam.api.dto.NamedList; +import org.apache.cloudstack.veeam.api.dto.Tag; +import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.cloudstack.veeam.utils.PathUtil; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.component.ManagerBase; + +public class TagsRouteHandler extends ManagerBase implements RouteHandler { + public static final String BASE_ROUTE = "/api/tags"; + + @Inject + ServerAdapter serverAdapter; + + @Override + public boolean start() { + return true; + } + + @Override + public int priority() { + return 5; + } + + @Override + public boolean canHandle(String method, String path) { + return getSanitizedPath(path).startsWith(BASE_ROUTE); + } + + @Override + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, + VeeamControlServlet io) throws IOException { + final String method = req.getMethod(); + if (!"GET".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET", outFormat); + return; + } + final String sanitizedPath = getSanitizedPath(path); + if (sanitizedPath.equals(BASE_ROUTE)) { + handleGet(req, resp, outFormat, io); + return; + } + + List idAndSubPath = PathUtil.extractIdAndSubPath(sanitizedPath, BASE_ROUTE); + if (CollectionUtils.isNotEmpty(idAndSubPath)) { + String id = idAndSubPath.get(0); + if (idAndSubPath.size() == 1) { + handleGetById(id, resp, outFormat, io); + return; + } + } + + io.notFound(resp, null, outFormat); + } + + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, + Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + final List result = serverAdapter.listAllTags(); + NamedList response = NamedList.of("tag", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } + + protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, + final VeeamControlServlet io) throws IOException { + try { + Tag response = serverAdapter.getTag(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 70a34ba08a64..eba432b7879d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -34,7 +34,6 @@ import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Nic; -import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.ResourceAction; import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -46,6 +45,7 @@ import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; @@ -108,7 +108,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path if (!"GET".equalsIgnoreCase(method) && !"PUT".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { io.methodNotAllowed(resp, "GET, PUT, DELETE", outFormat); } else if ("GET".equalsIgnoreCase(method)) { - handleGetById(id, resp, outFormat, io); + handleGetById(id, req, resp, outFormat, io); } else if ("PUT".equalsIgnoreCase(method)) { handleUpdateById(id, req, resp, outFormat, io); } else if ("DELETE".equalsIgnoreCase(method)) { @@ -308,10 +308,22 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons } } - protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleGetById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + String followStr = req.getParameter("follow"); + boolean includeDisks = false; + boolean includeNics = false; + if (StringUtils.isNotBlank(followStr)) { + Set followParts = java.util.Arrays.stream(followStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(java.util.stream.Collectors.toSet()); + includeDisks = followParts.contains("disk_attachments.disk"); + includeNics = followParts.contains("nics.reporteddevices"); + } + boolean allContent = Boolean.parseBoolean(req.getParameter("all_content")); try { - Vm response = serverAdapter.getInstance(id); + Vm response = serverAdapter.getInstance(id, includeDisks, includeNics, allContent); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -399,7 +411,7 @@ protected void handleGetNicsByVmId(final String id, final HttpServletResponse re final VeeamControlServlet io) throws IOException { try { List nics = serverAdapter.listNicsByInstanceUuid(id); - Nics response = new Nics(nics); + NamedList response = NamedList.of("nic", nics); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index bdae4983694e..49bf1f1cabad 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -92,7 +92,7 @@ protected static void fillAction(final ResourceAction action, final AsyncJobJoin public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { VmAction action = new VmAction(); fillAction(action, vo); - action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null)); + action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null, false)); return action; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java new file mode 100644 index 000000000000..d22a234d9e47 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.converter; + +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.TagsRouteHandler; +import org.apache.cloudstack.veeam.api.VmsRouteHandler; +import org.apache.cloudstack.veeam.api.dto.BaseDto; +import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Tag; + +import com.cloud.server.ResourceTag; +import com.cloud.tags.ResourceTagVO; + +public class ResourceTagVOToTagConverter { + + public static Ref getRootTagRef() { + String basePath = VeeamControlService.ContextPath.value(); + return Ref.of(basePath + TagsRouteHandler.BASE_ROUTE + "/" + BaseDto.ZERO_UUID, BaseDto.ZERO_UUID); + } + + public static Tag getRootTag() { + String basePath = VeeamControlService.ContextPath.value(); + Tag tag = new Tag(); + tag.setId(BaseDto.ZERO_UUID); + tag.setName("root"); + tag.setHref(getRootTagRef().getHref()); + return tag; + } + + public static Tag toTag(ResourceTagVO vo) { + String basePath = VeeamControlService.ContextPath.value(); + Tag tag = new Tag(); + tag.setId(vo.getUuid()); + tag.setName(vo.getKey()); + tag.setDescription(String.format("Tag %s-%s", vo.getKey(), vo.getValue())); + tag.setHref(basePath + TagsRouteHandler.BASE_ROUTE + "/" + vo.getUuid()); + if (ResourceTag.ResourceObjectType.UserVm.equals(vo.getResourceType())) { + tag.setVm(Ref.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vo.getResourceUuid(), + vo.getResourceUuid())); + } + tag.setParent(getRootTagRef()); + return tag; + } + + public static List toTags(List vos) { + return vos.stream().map(ResourceTagVOToTagConverter::toTag).collect(Collectors.toList()); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 36e1a04c4b47..6c7c8bddd794 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -27,14 +27,13 @@ import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.BaseDto; -import org.apache.cloudstack.veeam.api.dto.Bios; import org.apache.cloudstack.veeam.api.dto.Cpu; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; import org.apache.cloudstack.veeam.api.dto.EmptyElement; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Nic; -import org.apache.cloudstack.veeam.api.dto.Nics; import org.apache.cloudstack.veeam.api.dto.Os; +import org.apache.cloudstack.veeam.api.dto.OvfXmlUtil; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; @@ -55,7 +54,9 @@ private UserVmJoinVOToVmConverter() { * @param src UserVmJoinVO */ public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, - final Function> disksResolver, final Function> nicsResolver) { + final Function> disksResolver, + final Function> nicsResolver, + final boolean allContent) { if (src == null) { return null; } @@ -104,7 +105,13 @@ public static Vm toVm(final UserVmJoinVO src, final Function h } } - dst.setMemory(String.valueOf(src.getRamSize() * 1024L * 1024L)); + String memory = String.valueOf(src.getRamSize() * 1024L * 1024L); + dst.setMemory(memory); + Vm.MemoryPolicy memoryPolicy = new Vm.MemoryPolicy(); + memoryPolicy.setGuaranteed(memory); + memoryPolicy.setMax(memory); + memoryPolicy.setBallooning("false"); + dst.setMemoryPolicy(memoryPolicy); Cpu cpu = new Cpu(); cpu.setArchitecture(src.getArch()); cpu.setTopology(new Topology(src.getCpu(), 1, 1)); @@ -113,9 +120,15 @@ public static Vm toVm(final UserVmJoinVO src, final Function h os.setType(src.getGuestOsId() % 2 == 0 ? "windows" : "linux"); + Os.Boot boot = new Os.Boot(); + boot.setDevices(NamedList.of("device", List.of("hd"))); + os.setBoot(boot); dst.setOs(os); - Bios bios = new Bios(); + Vm.Bios bios = new Vm.Bios(); bios.setType("q35_secure_boot"); + Vm.Bios.BootMenu bootMenu = new Vm.Bios.BootMenu(); + bootMenu.setEnabled("false"); + bios.setBootMenu(bootMenu); dst.setBios(bios); dst.setType("desktop"); dst.setOrigin("ovirt"); @@ -126,9 +139,9 @@ public static Vm toVm(final UserVmJoinVO src, final Function h dst.setDiskAttachments(NamedList.of("disk_attachment", diskAttachments)); } - if (disksResolver != null) { + if (nicsResolver != null) { List nics = nicsResolver.apply(src); - dst.setNics(new Nics(nics)); + dst.setNics(NamedList.of("nic", nics)); } dst.setActions(NamedList.of("link", List.of( @@ -143,13 +156,29 @@ public static Vm toVm(final UserVmJoinVO src, final Function h BaseDto.getActionLink("snapshots", dst.getHref()) )); dst.setTags(new EmptyElement()); + dst.setCpuProfile(Ref.of( + basePath + ApiService.BASE_ROUTE + "/cpuprofiles/" + src.getServiceOfferingUuid(), + src.getServiceOfferingUuid())); + if (allContent) { + dst.setInitialization(getOvfInitialization(dst)); + } return dst; } + private static Vm.Initialization getOvfInitialization(Vm vm) { + final Vm.Initialization.Configuration configuration = new Vm.Initialization.Configuration(); + configuration.setType("ovf"); + configuration.setData(OvfXmlUtil.toXml(vm)); + + final Vm.Initialization initialization = new Vm.Initialization(); + initialization.setConfiguration(configuration); + return initialization; + } + public static List toVmList(final List srcList, final Function hostResolver) { return srcList.stream() - .map(v -> toVm(v, hostResolver, null, null)) + .map(v -> toVm(v, hostResolver, null, null, false)) .collect(Collectors.toList()); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java index 5ae2eb824224..5f98ca775dc2 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java @@ -22,6 +22,8 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class BaseDto { + public static final String ZERO_UUID = "00000000-0000-0000-0000-000000000000"; + private String href; private String id; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java deleted file mode 100644 index 6a354d5e749d..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BootMenu.java +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class BootMenu { - - private String enabled = "false"; - - public String getEnabled() { - return enabled; - } - - public void setEnabled(String enabled) { - this.enabled = enabled; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java deleted file mode 100644 index 0ded2f095f35..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HardwareInformation.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class HardwareInformation { - private String manufacturer; - private String productName; - private String serialNumber; - private String uuid; - private String version; - - public String getManufacturer() { - return manufacturer; - } - - public void setManufacturer(String manufacturer) { - this.manufacturer = manufacturer; - } - - public String getProductName() { - return productName; - } - - public void setProductName(String productName) { - this.productName = productName; - } - - public String getSerialNumber() { - return serialNumber; - } - - public void setSerialNumber(String serialNumber) { - this.serialNumber = serialNumber; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java index c937cdb564ba..8c4dba1d57c6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java @@ -259,4 +259,53 @@ public List getLink() { public void setLink(List link) { this.link = link; } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class HardwareInformation { + private String manufacturer; + private String productName; + private String serialNumber; + private String uuid; + private String version; + + public String getManufacturer() { + return manufacturer; + } + + public void setManufacturer(String manufacturer) { + this.manufacturer = manufacturer; + } + + public String getProductName() { + return productName; + } + + public void setProductName(String productName) { + this.productName = productName; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java deleted file mode 100644 index a1d4b4aa734d..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/HostSummary.java +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class HostSummary { - @JsonProperty("active") - private String active; - - @JsonProperty("migrating") - private String migrating; - - @JsonProperty("total") - private String total; - - public String getActive() { - return active; - } - - public void setActive(String active) { - this.active = active; - } - - public String getMigrating() { - return migrating; - } - - public void setMigrating(String migrating) { - this.migrating = migrating; - } - - public String getTotal() { - return total; - } - - public void setTotal(String total) { - this.total = total; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java deleted file mode 100644 index 1d1a46675015..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nics.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "nics") -public final class Nics { - - @JsonProperty("nic") - @JacksonXmlElementWrapper(useWrapping = false) - public List nic; - - public Nics() { - } - - public Nics(final List nic) { - this.nic = nic; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java index da73ebd9069b..af17151d4335 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java @@ -22,6 +22,8 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public final class Os { private String type; + private String version; + private Boot boot; public String getType() { return type; @@ -30,4 +32,24 @@ public String getType() { public void setType(String type) { this.type = type; } + + public Boot getBoot() { + return boot; + } + + public void setBoot(Boot boot) { + this.boot = boot; + } + + public final static class Boot { + private NamedList devices; + + public NamedList getDevices() { + return devices; + } + + public void setDevices(NamedList devices) { + this.devices = devices; + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java new file mode 100644 index 000000000000..3b0662b7c6b0 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -0,0 +1,671 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.dto; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; +import java.util.UUID; + +import javax.xml.XMLConstants; +import javax.xml.namespace.NamespaceContext; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class OvfXmlUtil { + + private static final String NS_OVF = "http://schemas.dmtf.org/ovf/envelope/1/"; + private static final String NS_RASD = "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"; + private static final String NS_VSSD = "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData"; + private static final String NS_XSI = "http://www.w3.org/2001/XMLSchema-instance"; + + private static final String ZERO_UUID = "00000000-0000-0000-0000-000000000000"; + private static final TimeZone UTC = TimeZone.getTimeZone("Etc/GMT"); + + private static final ThreadLocal OVIRT_DTF = ThreadLocal.withInitial(() -> { + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.ROOT); + sdf.setTimeZone(UTC); + return sdf; + }); + + public static String toXml(final Vm vm) { + final String vmId = vm.getId(); + final String vmName = vm.getName(); + final String vmDesc = defaultString(vm.getDescription()); + + final long creationMillis = vm.getCreationTime(); + final String creationDate = formatDate(creationMillis); + final String exportDate = formatDate(System.currentTimeMillis()); + final String stopTime = vm.getStopTime() != null ? formatDate(vm.getStopTime()) : creationDate; + final String bootTime = vm.getStartTime() != null ? formatDate(vm.getStartTime()) : creationDate; + + // Memory: Vm.memory is bytes (string) + final long memBytes = parseLong(vm.getMemory(), 1024L * 1024L * 1024L); + final long memMb = Math.max(128, memBytes / (1024L * 1024L)); + + // CPU: topology cores/sockets/threads. We default sockets=1 threads=1. + final int vcpu = Math.max(1, Integer.parseInt(vm.getCpu().getTopology().getCores())); + final int sockets = Math.max(1, Integer.parseInt(vm.getCpu().getTopology().getSockets())); + final int threads = Math.max(1, Integer.parseInt(vm.getCpu().getTopology().getThreads())); + final int cpuPerSocket = Math.max(1, vcpu / sockets); + final int maxVcpu = vcpu; + + // Template + final Ref template = vm.getTemplate(); + final String templateId = template != null && StringUtils.isNotBlank(template.getId()) ? template.getId() : ZERO_UUID; + final String templateName = template != null ? defaultString(template.getId()) : "Blank"; + + // Snapshot id (stable per VM id) + final String snapshotId = UUID.nameUUIDFromBytes(("ovf-snap-" + vmId).getBytes(StandardCharsets.UTF_8)).toString(); + + final StringBuilder sb = new StringBuilder(16_384); + sb.append(""); + sb.append(""); + + // --- References (from disks) --- + sb.append(""); + for (DiskAttachment da : diskAttachments(vm)) { + if (da == null || da.getDisk() == null || StringUtils.isBlank(da.getDisk().getId())) { + continue; + } + final String diskId = da.getDisk().getId(); + final String storageDomainId = firstStorageDomainId(da.getDisk()); + final String href = storageDomainId + "/" + diskId; + sb.append(""); + } + sb.append(""); + + // --- NetworkSection --- + sb.append(""); + sb.append("List of networks"); + // oVirt often lists networks, but can also be empty. We'll include known names if we can. + for (Nic nic : nics(vm)) { + if (nic == null) { + continue; + } + final String netName = inferNetworkName(nic); + if (StringUtils.isBlank(netName)) { + continue; + } + sb.append(""); + sb.append("").append(escapeText(defaultString(nic.getDescription()))).append(""); + sb.append(""); + } + sb.append(""); + + // --- DiskSection --- + sb.append("
"); + sb.append("List of Virtual Disks"); + for (DiskAttachment da : diskAttachments(vm)) { + if (da == null || da.getDisk() == null || StringUtils.isBlank(da.getDisk().getId())) { + continue; + } + final org.apache.cloudstack.veeam.api.dto.Disk d = da.getDisk(); + final String diskId = d.getId(); + final String storageDomainId = firstStorageDomainId(d); + final String href = storageDomainId + "/" + diskId; + final long provBytes = parseLong(d.getProvisionedSize(), 0); + final long actualBytes = parseLong(d.getActualSize(), 0); + final long provGiB = bytesToGibCeil(provBytes); + final long actualGiB = bytesToGibCeil(actualBytes); + final String diskInterface = mapDiskInterface(da.getIface()); + + sb.append(" 0 ? provGiB : 1).append("\""); + sb.append(" ovf:actual_size=\"").append(actualGiB > 0 ? actualGiB : 1).append("\""); + sb.append(" ovf:vm_snapshot_id=\"").append(escapeAttr(snapshotId)).append("\""); + sb.append(" ovf:parentRef=\"\""); + sb.append(" ovf:fileRef=\"").append(escapeAttr(href)).append("\""); + sb.append(" ovf:format=\"").append(escapeAttr(mapOvfDiskFormat(d.getFormat(), d.getSparse()))).append("\""); + sb.append(" ovf:volume-format=\"").append(escapeAttr(mapVolumeFormat(d.getFormat()))).append("\""); + sb.append(" ovf:volume-type=\"").append(escapeAttr(mapVolumeType(d.getSparse()))).append("\""); + sb.append(" ovf:disk-interface=\"").append(escapeAttr(diskInterface)).append("\""); + sb.append(" ovf:read-only=\"").append(escapeAttr(booleanString(da.getReadOnly(), "false"))).append("\""); + sb.append(" ovf:shareable=\"").append(escapeAttr(booleanString(d.getShareable(), "false"))).append("\""); + sb.append(" ovf:boot=\"").append(escapeAttr(booleanString(da.getBootable(), "false"))).append("\""); + sb.append(" ovf:pass-discard=\"").append(escapeAttr(booleanString(da.getPassDiscard(), "false"))).append("\""); + sb.append(" ovf:incremental-backup=\"false\""); + sb.append(" ovf:disk-alias=\"").append(escapeAttr(defaultString(d.getAlias()))).append("\""); + sb.append(" ovf:disk-description=\"").append(escapeAttr(defaultString(d.getDescription()))).append("\""); + sb.append(" ovf:wipe-after-delete=\"").append(escapeAttr(booleanString(d.getWipeAfterDelete(), "false"))).append("\""); + sb.append(">"); + } + sb.append("
"); + + // --- Content / VirtualSystem --- + sb.append(""); + sb.append("").append(escapeText(vmName)).append(""); + sb.append("").append(escapeText(vmDesc)).append(""); + sb.append(""); + sb.append("").append(creationDate).append(""); + sb.append("").append(exportDate).append(""); + sb.append("false"); + sb.append("guest_agent"); + sb.append("false"); + sb.append("1"); + sb.append("Etc/GMT"); + sb.append("0"); + sb.append("11"); + sb.append("4.8"); + sb.append("1"); + sb.append("AUTO_RESUME"); + sb.append("").append(memMb).append(""); + sb.append("").append(escapeText(booleanString(vm.getStateless(), "false"))).append(""); + sb.append("false"); + sb.append("false"); + sb.append("0"); + sb.append("").append(ZERO_UUID).append(""); + sb.append("0"); + sb.append("").append(escapeText(booleanString(vm.getBios() != null && vm.getBios().getBootMenu() != null ? vm.getBios().getBootMenu().getEnabled() : null, "false"))).append(""); + sb.append("true"); + sb.append("true"); + sb.append("false"); + sb.append("LOCK_SCREEN"); + sb.append("0"); + sb.append(""); + sb.append("").append(mapBiosType(vm.getBios() != null ? vm.getBios().getType() : null)).append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append("").append(memMb).append(""); + sb.append("true"); + sb.append("false"); + sb.append("false"); + sb.append("").append(mapBalloonEnabled(vm)).append(""); + sb.append("0"); + sb.append(""); + sb.append("").append(escapeText(templateId)).append(""); + sb.append("").append(escapeText(templateName)).append(""); + sb.append("true"); + sb.append("3"); + sb.append("").append(ZERO_UUID).append(""); + sb.append("2"); + sb.append("false"); + sb.append("").append(escapeText(templateId)).append(""); + sb.append("").append(escapeText(templateName)).append(""); + sb.append("false"); + sb.append("").append(stopTime).append(""); + sb.append("").append(bootTime).append(""); + sb.append("0"); + + // --- Operating system section --- + sb.append("
"); + sb.append("Guest Operating System"); + sb.append("").append(escapeText(inferOsDescription(vm))).append(""); + sb.append("
"); + + // --- Virtual hardware section --- + sb.append("
"); + sb.append("").append(vcpu).append(" CPU, ").append(memMb).append(" Memory"); + sb.append(""); + sb.append("ENGINE 4.4.0.0"); + sb.append(""); + + // CPU + sb.append(""); + sb.append("").append(vcpu).append(" virtual cpu"); + sb.append("Number of virtual CPU"); + sb.append("1"); + sb.append("3"); + sb.append("").append(sockets).append(""); + sb.append("").append(cpuPerSocket).append(""); + sb.append("").append(threads).append(""); + sb.append("").append(maxVcpu).append(""); + sb.append("").append(vcpu).append(""); + sb.append(""); + + // Memory + sb.append(""); + sb.append("").append(memMb).append(" MB of memory"); + sb.append("Memory Size"); + sb.append("2"); + sb.append("4"); + sb.append("MegaBytes"); + sb.append("").append(memMb).append(""); + sb.append(""); + + // Disks as Items + int diskUnit = 0; + for (DiskAttachment da : diskAttachments(vm)) { + if (da == null || da.getDisk() == null || StringUtils.isBlank(da.getDisk().getId())) { + continue; + } + final org.apache.cloudstack.veeam.api.dto.Disk d = da.getDisk(); + final String diskId = d.getId(); + final String storageDomainId = firstStorageDomainId(d); + final String href = storageDomainId + "/" + diskId; + + sb.append(""); + sb.append("").append(escapeText(defaultString(d.getAlias()))).append(""); + sb.append("").append(escapeText(diskId)).append(""); + sb.append("17"); + sb.append("").append(escapeText(href)).append(""); + sb.append("").append(ZERO_UUID).append(""); + sb.append("").append(escapeText(templateId)).append(""); + sb.append(""); + sb.append("").append(escapeText(storageDomainId)).append(""); + sb.append("").append(ZERO_UUID).append(""); + sb.append("").append(creationDate).append(""); + sb.append("").append(exportDate).append(""); + sb.append("").append(exportDate).append(""); + sb.append("disk"); + sb.append("disk"); + sb.append("").append(escapeText("{type=drive, bus=0, controller=0, target=0, unit=" + diskUnit + "}")).append(""); + sb.append("").append("true".equalsIgnoreCase(da.getBootable()) ? 1 : 0).append(""); + sb.append("true"); + sb.append("").append("true".equalsIgnoreCase(da.getReadOnly())).append(""); + sb.append("").append(escapeText("ua-" + href)).append(""); + sb.append(""); + diskUnit++; + } + + // NICs as Items + int nicSlot = 0; + for (Nic nic : nics(vm)) { + if (nic == null) { + continue; + } + final String nicId = firstNonBlank(nic.getId(), UUID.nameUUIDFromBytes(("nic-" + vmId + "-" + nicSlot).getBytes(StandardCharsets.UTF_8)).toString()); + final String nicName = firstNonBlank(nic.getName(), "nic" + (nicSlot + 1)); + final String mac = nic.getMac() != null ? defaultString(nic.getMac().getAddress()) : ""; + + sb.append(""); + sb.append("Ethernet adapter on [No Network]"); + sb.append("").append(escapeText(nicId)).append(""); + sb.append("10"); + sb.append(""); + sb.append("").append(mapNicResourceSubType(nic.getInterfaceType())).append(""); + sb.append("").append(escapeText(defaultString(inferNetworkName(nic)))).append(""); + sb.append("").append(escapeText(booleanString(nic.getLinked(), "true"))).append(""); + sb.append("").append(escapeText(nicName)).append(""); + sb.append("").append(escapeText(nicName)).append(""); + sb.append("").append(escapeText(mac)).append(""); + sb.append("10000"); + sb.append("interface"); + sb.append("bridge"); + sb.append("").append(escapeText("{type=pci, slot=0x" + String.format("%02x", nicSlot) + ", bus=0x01, domain=0x0000, function=0x0}")).append(""); + sb.append("0"); + sb.append("").append(escapeText(booleanString(nic.getPlugged(), "true"))).append(""); + sb.append("false"); + sb.append("").append(escapeText("ua-" + nicId)).append(""); + sb.append(""); + nicSlot++; + } + + // A few common devices that some consumers expect to exist (kept minimal) + // USB controller + sb.append(""); + sb.append("USB Controller"); + sb.append("3"); + sb.append("23"); + sb.append("DISABLED"); + sb.append(""); + + // RNG device + sb.append(""); + sb.append("0"); + sb.append("").append(UUID.nameUUIDFromBytes(("rng-" + vmId).getBytes(StandardCharsets.UTF_8))).append(""); + sb.append("rng"); + sb.append("virtio"); + sb.append("{type=pci, slot=0x00, bus=0x06, domain=0x0000, function=0x0}"); + sb.append("0"); + sb.append("true"); + sb.append("false"); + sb.append(""); + sb.append("urandom"); + sb.append(""); + + sb.append(""); + sb.append(""); + + return sb.toString(); + } + + public static void updateFromConfiguration(Vm vm) { + if (ObjectUtils.anyNull(vm.getInitialization(), + vm.getInitialization().getConfiguration(), + vm.getInitialization().getConfiguration().getData())) { + return; + } + OvfXmlUtil.updateFromXml(vm, vm.getInitialization().getConfiguration().getData()); + } + + protected static void updateFromXml(Vm vm, String ovfXml) { + if (vm == null || StringUtils.isBlank(ovfXml)) { + return; + } + try { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + DocumentBuilder db = dbf.newDocumentBuilder(); + Document doc = db.parse(new ByteArrayInputStream(ovfXml.getBytes(StandardCharsets.UTF_8))); + + XPathFactory xpf = XPathFactory.newInstance(); + XPath xpath = xpf.newXPath(); + + // Register namespace context for XPath + xpath.setNamespaceContext(new OvfNamespaceContext()); + + Node hwSection = (Node) xpath.evaluate( + "//*[local-name()='Section' and @*[local-name()='type']='ovf:VirtualHardwareSection_Type']", + doc, + XPathConstants.NODE + ); + + if (hwSection != null) { + // Memory + NodeList memItems = (NodeList) xpath.evaluate( + ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='4']]", + hwSection, + XPathConstants.NODESET + ); + if (memItems != null && memItems.getLength() > 0) { + Node memItem = memItems.item(0); + String memStr = childText(memItem, "VirtualQuantity"); + if (StringUtils.isNotBlank(memStr)) { + vm.setMemory(memStr); + } + } + + // CPU + NodeList cpuItems = (NodeList) xpath.evaluate( + ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='3']]", + hwSection, + XPathConstants.NODESET + ); + if (cpuItems != null && cpuItems.getLength() > 0) { + Node cpuItem = cpuItems.item(0); + String socketsStr = childText(cpuItem, "num_of_sockets"); + String coresStr = childText(cpuItem, "cpu_per_socket"); + String threadsStr = childText(cpuItem, "threads_per_cpu"); + + if (vm.getCpu() == null) { + vm.setCpu(new Cpu()); + } + if (vm.getCpu().getTopology() == null) { + vm.getCpu().setTopology(new Topology()); + } + + if (StringUtils.isNotBlank(socketsStr)) { + vm.getCpu().getTopology().setSockets(socketsStr); + } + if (StringUtils.isNotBlank(coresStr)) { + vm.getCpu().getTopology().setCores(coresStr); + } + if (StringUtils.isNotBlank(threadsStr)) { + vm.getCpu().getTopology().setThreads(threadsStr); + } + } + } + } catch (Exception e) { + // Ignore parsing errors and keep original VM configuration + } + } + + private static String xpathString(XPath xpath, Document doc, String expression) { + try { + String value = (String) xpath.evaluate(expression, doc, XPathConstants.STRING); + return StringUtils.isBlank(value) ? null : value.trim(); + } catch (XPathExpressionException e) { + return null; + } + } + + private static String childText(Node parent, String localName) { + if (parent == null || StringUtils.isBlank(localName)) { + return null; + } + NodeList children = parent.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (child.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ln = child.getLocalName(); + if (StringUtils.isBlank(ln)) { + ln = child.getNodeName(); + } + if (localName.equalsIgnoreCase(ln)) { + return StringUtils.trim(child.getTextContent()); + } + } + return null; + } + + private static List diskAttachments(Vm vm) { + if (vm.getDiskAttachments() == null) { + return List.of(); + } + return vm.getDiskAttachments().getItems(); + } + + private static List nics(Vm vm) { + if (vm.getNics() == null) { + return List.of(); + } + return vm.getNics().getItems(); + } + + private static String inferOsDescription(Vm vm) { + if (vm.getOs() == null) { + return "other"; + } + String t = vm.getOs().getType(); + if (StringUtils.isBlank(t)) { + return "other"; + } + if (t.toLowerCase(Locale.ROOT).contains("win")) { + return "windows"; + } + if (t.toLowerCase(Locale.ROOT).contains("linux")) { + return "linux"; + } + return t; + } + + private static String inferNetworkName(Nic nic) { + return "Network-" + nic.getId(); + } + + private static String firstStorageDomainId(Disk d) { + if (ObjectUtils.allNotNull(d, d.getStorageDomains()) && CollectionUtils.isNotEmpty(d.getStorageDomains().getItems())) { + return d.getStorageDomains().getItems().get(0).getId(); + } + return UUID.randomUUID().toString(); + } + + private static String mapDiskInterface(String iface) { + if (StringUtils.isBlank(iface)) { + return "VirtIO_SCSI"; + } + String v = iface.toLowerCase(Locale.ROOT); + if (v.contains("virtio") && v.contains("scsi")) { + return "VirtIO_SCSI"; + } + if (v.contains("virtio")) { + return "VirtIO"; + } + if (v.contains("ide")) { + return "IDE"; + } + if (v.contains("sata")) { + return "SATA"; + } + return iface; + } + + private static String mapOvfDiskFormat(String format, String sparse) { + if ("true".equalsIgnoreCase(sparse)) { + return "http://www.vmware.com/specifications/vmdk.html#sparse"; + } + return "http://www.vmware.com/specifications/vmdk.html#sparse"; + } + + private static String mapVolumeFormat(String format) { + if (StringUtils.isBlank(format)) { + return "RAW"; + } + String f = format.toLowerCase(Locale.ROOT); + if (f.contains("cow") || f.contains("qcow")) { + return "COW"; + } + if (f.contains("raw")) { + return "RAW"; + } + return format.toUpperCase(Locale.ROOT); + } + + private static String mapVolumeType(String sparse) { + return "true".equalsIgnoreCase(sparse) ? "Sparse" : "Preallocated"; + } + + private static int mapBiosType(String biosType) { + if (StringUtils.isBlank(biosType)) { + return 2; + } + String t = biosType.toLowerCase(Locale.ROOT); + if (t.contains("uefi") || t.contains("secure")) { + return 2; + } + return 0; + } + + private static String mapBalloonEnabled(Vm vm) { + if (vm.getMemoryPolicy() == null || vm.getMemoryPolicy().getBallooning() == null) { + return "true"; + } + return "true".equalsIgnoreCase(vm.getMemoryPolicy().getBallooning()) ? "true" : "false"; + } + + private static int mapNicResourceSubType(String iface) { + if (StringUtils.isBlank(iface)) { + return 3; + } + String v = iface.toLowerCase(Locale.ROOT); + if (v.contains("virtio")) { + return 3; + } + return 3; + } + + private static String booleanString(String v, String def) { + if (StringUtils.isBlank(v)) { + return def; + } + if ("true".equalsIgnoreCase(v)) { + return "true"; + } + if ("false".equalsIgnoreCase(v)) { + return "false"; + } + return def; + } + + private static String firstNonBlank(String... vals) { + for (String v : vals) { + if (StringUtils.isNotBlank(v)) { + return v; + } + } + return ""; + } + + private static String defaultString(String s) { + return s == null ? "" : s; + } + + private static long parseLong(String s, long def) { + if (StringUtils.isBlank(s)) { + return def; + } + try { + return Long.parseLong(s); + } catch (Exception ignored) { + return def; + } + } + + private static long bytesToGibCeil(long bytes) { + if (bytes <= 0) { + return 0; + } + final long gib = 1024L * 1024L * 1024L; + return (bytes + gib - 1) / gib; + } + + private static String formatDate(long epochMillis) { + return OVIRT_DTF.get().format(new Date(epochMillis)); + } + + private static String escapeText(String s) { + if (s == null) { + return ""; + } + return s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } + + private static String escapeAttr(String s) { + return escapeText(s); + } + + protected static class OvfNamespaceContext implements NamespaceContext { + @Override + public String getNamespaceURI(String prefix) { + if ("ovf".equals(prefix)) return NS_OVF; + if ("rasd".equals(prefix)) return NS_RASD; + if ("vssd".equals(prefix)) return NS_VSSD; + if ("xsi".equals(prefix)) return NS_XSI; + return XMLConstants.NULL_NS_URI; + } + @Override + public String getPrefix(String namespaceURI) { + return null; + } + @Override + public java.util.Iterator getPrefixes(String namespaceURI) { + return null; + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Tag.java similarity index 58% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Tag.java index ca68bfe475ab..1a9493160b6e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Bios.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Tag.java @@ -17,27 +17,41 @@ package org.apache.cloudstack.veeam.api.dto; -import com.fasterxml.jackson.annotation.JsonInclude; +public class Tag extends BaseDto { + private String name; + private String description; + private Ref parent; + private Ref vm; + + public String getName() { + return name; + } -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class Bios { + public void setName(String name) { + this.name = name; + } - private String type; // "uefi" or "bios" or whatever mapping you choose - private BootMenu bootMenu = new BootMenu(); + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } - public String getType() { - return type; + public Ref getParent() { + return parent; } - public void setType(String type) { - this.type = type; + public void setParent(Ref parent) { + this.parent = parent; } - public BootMenu getBootMenu() { - return bootMenu; + public Ref getVm() { + return vm; } - public void setBootMenu(BootMenu bootMenu) { - this.bootMenu = bootMenu; + public void setVm(Ref vm) { + this.vm = vm; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 9d18dcc22346..227845a37b09 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -42,6 +42,7 @@ public final class Vm extends BaseDto { private Ref cluster; private Ref host; private String memory; // bytes + private MemoryPolicy memoryPolicy; private Cpu cpu; private Os os; private Bios bios; @@ -53,8 +54,22 @@ public final class Vm extends BaseDto { private List link; // related resources private EmptyElement tags; // empty private NamedList diskAttachments; - private Nics nics; - private VmInitialization initialization; + private NamedList nics; + private Initialization initialization; + + private Ref cpuProfile; + + public EmptyElement io = new EmptyElement(); + public EmptyElement migration = new EmptyElement(); + public EmptyElement sso = new EmptyElement(); + public EmptyElement usb = new EmptyElement(); + public EmptyElement quota = new EmptyElement(); + public EmptyElement highAvailability = new EmptyElement(); + public EmptyElement largeIcon = new EmptyElement(); + public EmptyElement smallIcon = new EmptyElement(); + public EmptyElement placementPolicy = new EmptyElement(); + public EmptyElement timeZone = new EmptyElement(); + public EmptyElement display = new EmptyElement(); public String getName() { return name; @@ -152,6 +167,14 @@ public void setMemory(String memory) { this.memory = memory; } + public MemoryPolicy getMemoryPolicy() { + return memoryPolicy; + } + + public void setMemoryPolicy(MemoryPolicy memoryPolicy) { + this.memoryPolicy = memoryPolicy; + } + public Cpu getCpu() { return cpu; } @@ -232,22 +255,145 @@ public void setDiskAttachments(NamedList diskAttachments) { this.diskAttachments = diskAttachments; } - public Nics getNics() { + public NamedList getNics() { return nics; } - public void setNics(Nics nics) { + public void setNics(NamedList nics) { this.nics = nics; } - public VmInitialization getInitialization() { + public Initialization getInitialization() { return initialization; } - public void setInitialization(VmInitialization initialization) { + public void setInitialization(Initialization initialization) { this.initialization = initialization; } + public Ref getCpuProfile() { + return cpuProfile; + } + + public void setCpuProfile(Ref cpuProfile) { + this.cpuProfile = cpuProfile; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class Bios { + + private String type; // "uefi" or "bios" or whatever mapping you choose + private BootMenu bootMenu = new BootMenu(); + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public BootMenu getBootMenu() { + return bootMenu; + } + + public void setBootMenu(BootMenu bootMenu) { + this.bootMenu = bootMenu; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class BootMenu { + + private String enabled; + + public String getEnabled() { + return enabled; + } + + public void setEnabled(String enabled) { + this.enabled = enabled; + } + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static final class MemoryPolicy { + + private String guaranteed; + private String max; + private String ballooning; + + public String getGuaranteed() { + return guaranteed; + } + + public void setGuaranteed(String guaranteed) { + this.guaranteed = guaranteed; + } + + public String getMax() { + return max; + } + + public void setMax(String max) { + this.max = max; + } + + public String getBallooning() { + return ballooning; + } + + public void setBallooning(String ballooning) { + this.ballooning = ballooning; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Initialization { + + private String customScript; + private Configuration configuration; + + public String getCustomScript() { + return customScript; + } + + public void setCustomScript(String customScript) { + this.customScript = customScript; + } + + public Configuration getConfiguration() { + return configuration; + } + + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class Configuration { + + private String data; + private String type; + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + } + } + public static Vm of(String href, String id) { Vm vm = new Vm(); vm.setHref(href); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java deleted file mode 100644 index a9e77b01a1cc..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmInitialization.java +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class VmInitialization { - - private String customScript; - - public String getCustomScript() { - return customScript; - } - - public void setCustomScript(String customScript) { - this.customScript = customScript; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index f56a19d84715..cbe11724648b 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -43,6 +43,7 @@ + diff --git a/plugins/integrations/veeam-control-service/src/main/resources/test.xml b/plugins/integrations/veeam-control-service/src/main/resources/test.xml new file mode 100644 index 000000000000..8d39bd424807 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/resources/test.xml @@ -0,0 +1,618 @@ + + + + + + + List of networks + +
+ List of Virtual Disks + +
+ + test-vm-abhisar + + + 2026/01/07 13:37:09 + 2026/01/08 04:07:00 + false + guest_agent + false + 1 + Etc/GMT + 0 + 11 + 4.8 + 1 + AUTO_RESUME + 1024 + false + false + false + 0 + c067a148-e4d5-11f0-98ce-00163e6c35f4 + 0 + false + true + true + false + LOCK_SCREEN + 0 + + 2 + + + + 4096 + true + false + false + true + 0 + Default + 00000000-0000-0000-0000-000000000000 + Blank + true + 3 + 95e46398-e4d5-11f0-bb71-00163e6c35f4 + 2 + false + 00000000-0000-0000-0000-000000000000 + Blank + false + 2026/01/07 13:37:09 + 2026/01/07 13:38:03 + 0 +
+ Guest Operating System + other +
+
+ 1 CPU, 1024 Memory + + ENGINE 4.4.0.0 + + + 1 virtual cpu + Number of virtual CPU + 1 + 3 + 1 + 1 + 1 + 16 + 1 + + + 1024 MB of memory + Memory Size + 2 + 4 + MegaBytes + 1024 + + + test-vm-abhisar_Disk1 + 5cbc2ed5-de89-44a4-aa58-b7161f8afaf8 + 17 + ddf18375-4c69-4ec5-8371-6dabc94e4e60/5cbc2ed5-de89-44a4-aa58-b7161f8afaf8 + 00000000-0000-0000-0000-000000000000 + 00000000-0000-0000-0000-000000000000 + + 41609681-c92a-410a-bcc2-5b5e1305cdd1 + 91f4d826-e4d5-11f0-bd93-00163e6c35f4 + 2026/01/07 13:36:59 + 2026/01/07 13:53:36 + 2026/01/08 04:07:00 + disk + disk + {type=drive, bus=0, controller=0, target=0, unit=0} + 1 + true + false + ua-ddf18375-4c69-4ec5-8371-6dabc94e4e60 + + + Ethernet adapter on [No Network] + 9a6f804d-b305-41db-b1b4-bdfd82c4b446 + 10 + + 3 + + true + nic1 + nic1 + 56:6f:9f:c0:00:07 + 10000 + interface + bridge + {type=pci, slot=0x00, bus=0x01, domain=0x0000, function=0x0} + 0 + true + false + ua-9a6f804d-b305-41db-b1b4-bdfd82c4b446 + + + USB Controller + 3 + 23 + DISABLED + + + Graphical Controller + 0d4a490c-f9d7-45dd-8686-69d5bae218d6 + 20 + 1 + false + video + vga + {type=pci, slot=0x01, bus=0x00, domain=0x0000, function=0x0} + 0 + true + false + ua-0d4a490c-f9d7-45dd-8686-69d5bae218d6 + + 16384 + + + + Graphical Framebuffer + f62554f1-05fe-472e-a34b-9e6b980ad59f + 26 + graphics + vnc + + 0 + true + false + + + + CDROM + 9c38cc6a-9def-46f3-bf1c-2b3f4aa6b764 + 15 + disk + cdrom + {type=drive, bus=0, controller=0, target=0, unit=2} + 0 + true + true + ua-9c38cc6a-9def-46f3-bf1c-2b3f4aa6b764 + + + + + + 0 + a737450e-20b5-427e-a18b-85ec20683e31 + channel + unix + {type=virtio-serial, bus=0, controller=0, port=1} + 0 + true + false + channel0 + + + 0 + 1d3ba276-9e8d-4a16-9cdf-dfd25180b7bc + channel + unix + {type=virtio-serial, bus=0, controller=0, port=2} + 0 + true + false + channel1 + + + 0 + 8f21ce42-9499-4ded-88d4-04dff2fdc3ff + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x0, multifunction=on} + 0 + true + false + pci.1 + + 1 + pcie-root-port + + + + 0 + d1b9d421-1a57-469d-97fe-0682ad4594c3 + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x1} + 0 + true + false + pci.2 + + 2 + pcie-root-port + + + + 0 + 768c4772-eb7a-4f0f-85a7-2b94e20fe78c + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x2} + 0 + true + false + pci.3 + + 3 + pcie-root-port + + + + 0 + d20bae3b-f5d7-4131-b00a-3cf66f390434 + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x3} + 0 + true + false + pci.4 + + 4 + pcie-root-port + + + + 0 + 5887f3ad-c575-488e-9138-fca9c7064ae5 + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x4} + 0 + true + false + pci.5 + + 5 + pcie-root-port + + + + 0 + f880f086-227e-4e25-b2fc-8a3d13d1f1bd + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x5} + 0 + true + false + pci.6 + + 6 + pcie-root-port + + + + 0 + d64f62a0-6176-482b-8d24-f82fb32b8f12 + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x6} + 0 + true + false + pci.7 + + 7 + pcie-root-port + + + + 0 + 1544f32e-1e94-4e10-b198-7c5e95ab280d + controller + pci + {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x7} + 0 + true + false + pci.8 + + 8 + pcie-root-port + + + + 0 + 7dd5080f-8c04-4593-8c6a-1dc5cd6c3e3e + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x0, multifunction=on} + 0 + true + false + pci.9 + + 9 + pcie-root-port + + + + 0 + 4dab4257-2729-482c-b4e1-6a3c05161153 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x1} + 0 + true + false + pci.10 + + 10 + pcie-root-port + + + + 0 + 99effa2f-2963-4abd-9eab-1cbe8e913ca4 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x2} + 0 + true + false + pci.11 + + 11 + pcie-root-port + + + + 0 + 2a376983-897b-4396-be32-89f2a9ca7d22 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x3} + 0 + true + false + pci.12 + + 12 + pcie-root-port + + + + 0 + 2e763d82-4475-4268-bc0a-07c915ec19c8 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x4} + 0 + true + false + pci.13 + + 13 + pcie-root-port + + + + 0 + ef39155f-760e-4374-afb9-ff05cc8b9609 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x5} + 0 + true + false + pci.14 + + 14 + pcie-root-port + + + + 0 + 74be06f0-84b6-472e-a054-486343f66084 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x6} + 0 + true + false + pci.15 + + 15 + pcie-root-port + + + + 0 + c68db43a-fa3a-4689-941d-b477d2676d27 + controller + pci + {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x7} + 0 + true + false + pci.16 + + 16 + pcie-root-port + + + + 0 + d11cbe26-ee82-4e15-b8eb-2aa7b285d00d + controller + pci + {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x0, multifunction=on} + 0 + true + false + pci.17 + + 17 + pcie-root-port + + + + 0 + c2ef6c73-f633-41c1-8736-7e9c8d748ac2 + controller + pci + {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x1} + 0 + true + false + pci.18 + + 18 + pcie-root-port + + + + 0 + 5944d260-08c3-4f12-aa22-1e9ac76ae6c0 + controller + pci + {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x2} + 0 + true + false + pci.19 + + 19 + pcie-root-port + + + + 0 + 8c7ad6aa-ac22-4d98-86b7-45f3a13c98da + controller + pci + {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x3} + 0 + true + false + pci.20 + + 20 + pcie-root-port + + + + 0 + dc1cfae5-682d-4bb5-a53e-d604852e62cd + controller + pci + {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x4} + 0 + true + false + pci.21 + + 21 + pcie-root-port + + + + 0 + 6117753b-8ce6-4568-8e09-c8b686396334 + controller + sata + {type=pci, slot=0x1f, bus=0x00, domain=0x0000, function=0x2} + 0 + true + false + ide + + 0 + + + + 0 + 17976687-41f8-4f7c-97f5-a76a282c40e4 + controller + virtio-serial + {type=pci, slot=0x00, bus=0x03, domain=0x0000, function=0x0} + 0 + true + false + ua-17976687-41f8-4f7c-97f5-a76a282c40e4 + + + 0 + 97f6991c-e4d5-11f0-9b4a-00163e6c35f4 + rng + virtio + {type=pci, slot=0x00, bus=0x06, domain=0x0000, function=0x0} + 0 + true + false + ua-97f6991c-e4d5-11f0-9b4a-00163e6c35f4 + + urandom + + + + 0 + 0eb75625-9891-4b03-9541-c58c43c323b2 + controller + virtio-scsi + {type=pci, slot=0x00, bus=0x02, domain=0x0000, function=0x0} + 0 + true + false + ua-0eb75625-9891-4b03-9541-c58c43c323b2 + + + + + + 0 + 59536909-bac6-4202-b2ad-d84a22a41013 + balloon + memballoon + {type=pci, slot=0x00, bus=0x05, domain=0x0000, function=0x0} + 0 + true + true + ua-59536909-bac6-4202-b2ad-d84a22a41013 + + virtio + + + + 0 + e95647b0-4bb2-4ccb-b867-cbde06311038 + controller + usb + {type=pci, slot=0x00, bus=0x04, domain=0x0000, function=0x0} + 0 + true + false + ua-e95647b0-4bb2-4ccb-b867-cbde06311038 + + 0 + qemu-xhci + + +
+
+ + ACTIVE + Active VM + 2026/01/07 13:37:09 + +
+ + \ No newline at end of file diff --git a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java new file mode 100644 index 000000000000..c01e19515fe5 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java @@ -0,0 +1,28 @@ +package org.apache.cloudstack.veeam.api.dto; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class OvfXmlUtilTest { + + String configuration = "" + + "adm-v9adm-v9"+ + "
1 CPU, 512 MemoryENGINE 4.4.0.01 virtual cpuNumber of virtual CPU1311111" + + "512 MB of memoryMemory Size24MegaBytes512" + + "
"; + + @Test + public void updateFromXml_parsesDetails() { + Vm vm = new Vm(); + OvfXmlUtil.updateFromXml(vm, configuration); + + assertEquals(String.valueOf(512L), vm.getMemory()); + assertEquals("1", vm.getCpu().getTopology().getSockets()); + assertEquals("1", vm.getCpu().getTopology().getCores()); + assertEquals("1", vm.getCpu().getTopology().getThreads()); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index ed44ded22805..e8390c8536b1 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -137,6 +137,10 @@ public Backup createBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("VM must be running or stopped to start backup"); } + if (vm.getBackupOfferingId() == null) { + throw new CloudRuntimeException("VM not assigned a backup offering"); + } + Backup existingBackup = backupDao.findByVmId(vmId); if (existingBackup != null && existingBackup.getStatus() == Backup.Status.BackingUp) { throw new CloudRuntimeException("Backup already in progress for VM: " + vmId); From 27a2eb0869453651643215f6d18a3aab4357d137 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 25 Feb 2026 17:40:15 +0530 Subject: [PATCH 045/129] fix Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/veeam/adapter/ServerAdapter.java | 4 ++-- .../cloudstack/backup/IncrementalBackupServiceImpl.java | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index eb2f94628fd3..c816ef41f31c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1193,7 +1193,7 @@ private BackupVO getBackupFromJob(ApiServerService.AsyncCmdResult result, UserVm } public Backup getBackup(String uuid) { - BackupVO vo = backupDao.findByUuid(uuid); + BackupVO vo = backupDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); } @@ -1233,7 +1233,7 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { if (result == null) { throw new CloudRuntimeException("Failed to finalize backup"); } - backup = backupDao.findById(backup.getId()); + backup = backupDao.findByIdIncludingRemoved(backup.getId()); return BackupVOToBackupConverter.toBackup(backup, id -> vm, this::getHostById, this::getBackupDisks); } catch (Exception e) { throw new CloudRuntimeException("Failed to finalize backup: " + e.getMessage(), e); diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index e8390c8536b1..d624af5322b5 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -160,7 +160,7 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setStatus(Backup.Status.BackingUp); + backup.setStatus(Backup.Status.ReadyForTransfer); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setDate(new Date()); @@ -230,8 +230,6 @@ public Backup startBackup(StartBackupCmd cmd) { // todo: set it in the backend backup.setType("Incremental"); } - backup.setStatus(Backup.Status.ReadyForTransfer); - backupDao.update(backup.getId(), backup); return backup; } From 18fbf76ba418c719ebb3f6d8a18588a39f3c41fe Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 26 Feb 2026 15:56:13 +0530 Subject: [PATCH 046/129] fix Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/veeam/api/dto/OvfXmlUtil.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java index 3b0662b7c6b0..a5e2da83c4dc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -359,12 +359,15 @@ public static String toXml(final Vm vm) { } public static void updateFromConfiguration(Vm vm) { - if (ObjectUtils.anyNull(vm.getInitialization(), - vm.getInitialization().getConfiguration(), - vm.getInitialization().getConfiguration().getData())) { + Vm.Initialization initialization = vm.getInitialization(); + if (initialization == null) { return; } - OvfXmlUtil.updateFromXml(vm, vm.getInitialization().getConfiguration().getData()); + Vm.Initialization.Configuration configuration = vm.getInitialization().getConfiguration(); + if (configuration == null) { + return; + } + OvfXmlUtil.updateFromXml(vm, configuration.getData()); } protected static void updateFromXml(Vm vm, String ovfXml) { From 11592b0ddc2ba7eec1c895283fbd6751db94e02c Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:47:17 +0530 Subject: [PATCH 047/129] fix image_server.py --- scripts/vm/hypervisor/kvm/image_server.py | 98 +---------------------- 1 file changed, 2 insertions(+), 96 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/image_server.py b/scripts/vm/hypervisor/kvm/image_server.py index 38400cdf2211..5119b8b7e6ac 100644 --- a/scripts/vm/hypervisor/kvm/image_server.py +++ b/scripts/vm/hypervisor/kvm/image_server.py @@ -439,94 +439,6 @@ def flush(self) -> None: return raise RuntimeError("libnbd binding has no flush/fsync method") - def get_zero_extents(self) -> List[Dict[str, Any]]: - """ - Query NBD block status (base:allocation) and return extents that are - hole or zero in imageio format: [{"start": ..., "length": ..., "zero": true}, ...]. - Returns [] if block status is not supported; fallback to one full-image - zero extent when we have size but block status fails. - """ - size = self.size() - if size == 0: - return [] - - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - logging.error("get_zero_extents: no block_status/block_status_64") - return self._fallback_zero_extent(size) - if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( - "base:allocation" - ): - logging.error( - "get_zero_extents: server did not negotiate base:allocation" - ) - return self._fallback_zero_extent(size) - - zero_extents: List[Dict[str, Any]] = [] - chunk = min(size, 64 * 1024 * 1024) # 64 MiB - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - # Binding typically passes (metacontext, offset, entries[, nr_entries][, error]). - metacontext = None - off = 0 - entries = None - if len(args) >= 3: - metacontext, off, entries = args[0], args[1], args[2] - else: - for a in args: - if isinstance(a, str): - metacontext = a - elif isinstance(a, int): - off = a - elif a is not None and hasattr(a, "__iter__"): - entries = a - if metacontext != "base:allocation" or entries is None: - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - if (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0: - zero_extents.append( - {"start": current, "length": length, "zero": True} - ) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return self._fallback_zero_extent(size) - - try: - while offset < size: - count = min(chunk, size - offset) - # Try (count, offset, callback) then (offset, count, callback) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.error("get_zero_extents block_status failed: %r", e) - return self._fallback_zero_extent(size) - if not zero_extents: - return self._fallback_zero_extent(size) - return zero_extents - - def _fallback_zero_extent(self, size: int) -> List[Dict[str, Any]]: - """Return one zero extent covering the whole image when block status unavailable.""" - return [{"start": 0, "length": size, "zero": True}] - def get_allocation_extents(self) -> List[Dict[str, Any]]: """ Query base:allocation and return all extents (allocated and hole/zero) @@ -694,6 +606,7 @@ def __exit__(self, exc_type, exc, tb) -> None: class Handler(BaseHTTPRequestHandler): server_version = "imageio-poc/0.1" + server_protocol = "HTTP/1.1" # Keep BaseHTTPRequestHandler from printing noisy default logs def log_message(self, fmt: str, *args: Any) -> None: @@ -1093,13 +1006,7 @@ def do_PATCH(self) -> None: def _handle_get_image( self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] ) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - if not _READ_SEM.acquire(blocking=False): - lock.release() self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") return @@ -1213,7 +1120,6 @@ def _handle_get_image( pass finally: _READ_SEM.release() - lock.release() dur = _now_s() - start logging.info( "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur @@ -1340,7 +1246,7 @@ def _handle_get_extents( cfg.get("export"), need_block_status=True, ) as conn: - extents = conn.get_zero_extents() + extents = conn.get_allocation_extents() self._send_json(HTTPStatus.OK, extents) except Exception as e: logging.error("EXTENTS error image_id=%s err=%r", image_id, e) From 8655f616313148099fa54f0b0cfccf21b52f5527 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 27 Feb 2026 09:13:46 +0530 Subject: [PATCH 048/129] fix pre-commit --- .../apache/cloudstack/backup/IncrementalBackupServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index d624af5322b5..a2a5f50d7afd 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -43,8 +43,6 @@ import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; From 196dd7fb28c2f6c6640bb3293a1ffdd9fed59784 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 27 Feb 2026 11:53:32 +0530 Subject: [PATCH 049/129] fix ovf end tag Signed-off-by: Abhishek Kumar --- .../java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java index a5e2da83c4dc..b4bc8517a800 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -352,6 +352,7 @@ public static String toXml(final Vm vm) { sb.append("urandom"); sb.append(""); + sb.append("
"); sb.append(""); sb.append(""); From 0dadbadb5286688010d6a6309da031c835b27c8f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 27 Feb 2026 13:17:54 +0530 Subject: [PATCH 050/129] fix start nbd server Signed-off-by: Abhishek Kumar --- .../cloudstack/backup/IncrementalBackupServiceImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index a2a5f50d7afd..fa5298e1fa85 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -386,8 +386,16 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { private void startNBDServer(String transferId, String direction, Long hostId, String exportName, String volumePath) { StartNBDServerAnswer nbdServerAnswer; + if (hostId == null) { + throw new CloudRuntimeException("Host cannot be determined for starting NBD server"); + } + HostVO host = hostDao.findById(hostId); + if (host == null) { + throw new CloudRuntimeException("Host cannot be found for starting NBD server with ID: " + hostId); + } StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, + host.getPublicIpAddress(), exportName, volumePath, transferId, From b68e541b31f183db419685a4c17e2a6bc236186f Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:29:06 +0530 Subject: [PATCH 051/129] remove hostIpaddress from startNbdCommand --- .../cloudstack/backup/StartNBDServerCommand.java | 14 ++------------ .../LibvirtStartNBDServerCommandWrapper.java | 4 ---- .../backup/IncrementalBackupServiceImpl.java | 1 - 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java index b0e452df33c6..47dd2b4a6df9 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -21,7 +21,6 @@ public class StartNBDServerCommand extends Command { private String transferId; - private String hostIpAddress; private String exportName; private String volumePath; private String socket; @@ -30,19 +29,14 @@ public class StartNBDServerCommand extends Command { public StartNBDServerCommand() { } - protected StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, String direction) { + protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction) { this.transferId = transferId; - this.hostIpAddress = hostIpAddress; + this.socket = socket; this.exportName = exportName; this.volumePath = volumePath; this.direction = direction; } - public StartNBDServerCommand(String transferId, String hostIpAddress, String exportName, String volumePath, String socket, String direction) { - this(transferId, hostIpAddress, exportName, volumePath, direction); - this.socket = socket; - } - public String getExportName() { return exportName; } @@ -51,10 +45,6 @@ public String getSocket() { return socket; } - public String getHostIpAddress() { - return hostIpAddress; - } - public String getTransferId() { return transferId; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index 71d9a06a3605..263e5f1cae5f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -39,7 +39,6 @@ public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper Date: Mon, 2 Mar 2026 10:33:46 +0530 Subject: [PATCH 052/129] image server : support for range puts and blocking writes --- scripts/vm/hypervisor/kvm/image_server.py | 251 ++++++++++++++++++---- 1 file changed, 205 insertions(+), 46 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/image_server.py b/scripts/vm/hypervisor/kvm/image_server.py index 5119b8b7e6ac..891bac5bf53b 100644 --- a/scripts/vm/hypervisor/kvm/image_server.py +++ b/scripts/vm/hypervisor/kvm/image_server.py @@ -20,8 +20,12 @@ POC "imageio-like" HTTP server backed by NBD over Unix socket or a local file. Supports two backends (see config payload): -- nbd: proxy to an NBD server via Unix socket (socket path, export, export_bitmap); supports range reads/writes, extents, zero, flush. -- file: read/write a local qcow2 (or raw) file path; full PUT only (no range writes), GET with optional ranges, flush. +- nbd: proxy to an NBD server via Unix socket (socket path, export, export_bitmap); + supports range reads/writes (GET/PUT/PATCH), extents, zero, flush. PUT accepts + full upload or ranged upload (Content-Range). Concurrent PUTs on the same image + are serialized (blocking). +- file: read/write a local qcow2 (or raw) file path; full PUT only (no range + writes), GET with optional ranges, flush. How to run ---------- @@ -44,8 +48,16 @@ - GET a byte range: curl -v -H "Range: bytes=0-1048575" http://127.0.0.1:54323/images/demo -o first_1MiB.bin -- PUT full image (Content-Length must equal export size exactly): +- PUT full image (Content-Length must equal export size exactly). Optional ?flush=y|n: curl -v -T demo.img http://127.0.0.1:54323/images/demo + curl -v -T demo.img "http://127.0.0.1:54323/images/demo?flush=y" + +- PUT ranged (NBD backend only). Content-Range: bytes start-end/* or bytes start-end/size + (server uses only start; length from Content-Length). Optional ?flush=y|n: + curl -v -X PUT -H "Content-Range: bytes 0-1048575/*" -H "Content-Length: 1048576" \ + --data-binary @chunk.bin http://127.0.0.1:54323/images/demo + curl -v -X PUT -H "Content-Range: bytes 1048576-2097151/52428800" -H "Content-Length: 1048576" \ + --data-binary @chunk2.bin "http://127.0.0.1:54323/images/demo?flush=n" - GET extents (zero/hole extents from NBD base:allocation): curl -s http://127.0.0.1:54323/images/demo/extents | jq . @@ -89,6 +101,7 @@ import json import logging import os +import re import socket import threading import time @@ -608,6 +621,9 @@ class Handler(BaseHTTPRequestHandler): server_version = "imageio-poc/0.1" server_protocol = "HTTP/1.1" + # Accept both "bytes start-end/*" and "bytes start-end/size"; we only use start. + _CONTENT_RANGE_RE = re.compile(r"^bytes\s+(\d+)-(\d+)/(?:\*|\d+)$") + # Keep BaseHTTPRequestHandler from printing noisy default logs def log_message(self, fmt: str, *args: Any) -> None: logging.info("%s - - %s", self.address_string(), fmt % args) @@ -740,6 +756,23 @@ def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: return None, None return image_id, tail + def _parse_content_range(self, header: str) -> Tuple[int, int]: + """ + Parse Content-Range header "bytes start-end/*" or "bytes start-end/size" + and return (start, end_inclusive). Raises ValueError on invalid input. + """ + if not header: + raise ValueError("empty Content-Range") + m = self._CONTENT_RANGE_RE.match(header.strip()) + if not m: + raise ValueError("invalid Content-Range") + start_s, end_s = m.groups() + start = int(start_s, 10) + end = int(end_s, 10) + if start < 0 or end < start: + raise ValueError("invalid Content-Range range") + return start, end + def _parse_query(self) -> Dict[str, List[str]]: """Parse query string from self.path into a dict of name -> list of values.""" if "?" not in self.path: @@ -855,9 +888,11 @@ def do_PUT(self) -> None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - if self.headers.get("Range") is not None or self.headers.get("Content-Range") is not None: + # For PUT we only support Content-Range (for NBD backend); Range is rejected. + if self.headers.get("Range") is not None: self._send_error_json( - HTTPStatus.BAD_REQUEST, "Range/Content-Range not supported; full writes only" + HTTPStatus.BAD_REQUEST, + "Range header not supported for PUT; use Content-Range or PATCH", ) return @@ -874,7 +909,24 @@ def do_PUT(self) -> None: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") return - self._handle_put_image(image_id, cfg, content_length) + # Optional flush=y|n query parameter. + query = self._parse_query() + flush_param = (query.get("flush") or ["n"])[0].lower() + flush = flush_param in ("y", "yes", "true", "1") + + content_range_hdr = self.headers.get("Content-Range") + if content_range_hdr is not None: + if self._is_file_backend(cfg): + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "Content-Range PUT not supported for file backend; use full PUT", + ) + return + self._handle_put_range(image_id, cfg, content_range_hdr, content_length, flush) + return + + # No Content-Range: full image PUT. + self._handle_put_image(image_id, cfg, content_length, flush) def do_POST(self) -> None: image_id, tail = self._parse_route() @@ -1004,7 +1056,7 @@ def do_PATCH(self) -> None: self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) def _handle_get_image( - self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] + self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] ) -> None: if not _READ_SEM.acquire(blocking=False): self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") @@ -1125,11 +1177,12 @@ def _handle_get_image( "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur ) - def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: int) -> None: + def _handle_put_image( + self, image_id: str, cfg: Dict[str, Any], content_length: int, flush: bool + ) -> None: lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return + # Block until we can write this image + lock.acquire() if not _WRITE_SEM.acquire(blocking=False): lock.release() @@ -1155,7 +1208,13 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: f.write(chunk) bytes_written += len(chunk) remaining -= len(chunk) - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + if flush: + f.flush() + os.fsync(f.fileno()) + self._send_json( + HTTPStatus.OK, + {"ok": True, "bytes_written": bytes_written, "flushed": flush}, + ) else: with _NbdConn(cfg["socket"], cfg.get("export")) as conn: offset = 0 @@ -1173,8 +1232,12 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: remaining -= len(chunk) bytes_written += len(chunk) - # POC-level: do not auto-flush on PUT; expose explicit /flush endpoint. - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + if flush: + conn.flush() + self._send_json( + HTTPStatus.OK, + {"ok": True, "bytes_written": bytes_written, "flushed": flush}, + ) except Exception as e: logging.error("PUT error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") @@ -1186,6 +1249,49 @@ def _handle_put_image(self, image_id: str, cfg: Dict[str, Any], content_length: "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur ) + def _write_range_nbd( + self, + image_id: str, + cfg: Dict[str, Any], + start_off: int, + content_length: int, + ) -> Tuple[int, bool]: + """ + Low-level helper: write request body to NBD backend starting at start_off. + The length is taken from Content-Length. Returns (bytes_written, ok). + If ok is False, an error response was already sent. + """ + bytes_written = 0 + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + image_size = conn.size() + if start_off >= image_size: + self._send_range_not_satisfiable(image_size) + return 0, False + + # Clamp to image size: we do not allow writes beyond end of image. + max_len = image_size - start_off + if content_length > max_len: + self._send_range_not_satisfiable(image_size) + return 0, False + + offset = start_off + remaining = content_length + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"request body ended early at {bytes_written} bytes", + ) + return bytes_written, False + conn.pwrite(chunk, offset) + n = len(chunk) + offset += n + remaining -= n + bytes_written += n + + return bytes_written, True + def _handle_get_extents( self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None ) -> None: @@ -1356,40 +1462,28 @@ def _handle_patch_range( ) with _NbdConn(cfg["socket"], cfg.get("export")) as conn: image_size = conn.size() - try: - start_off, end_inclusive = self._parse_single_range( - range_header, image_size - ) - except ValueError as e: - if "unsatisfiable" in str(e).lower(): - self._send_range_not_satisfiable(image_size) - else: - self._send_error_json( - HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" - ) - return - expected_len = end_inclusive - start_off + 1 - if content_length != expected_len: + try: + start_off, end_inclusive = self._parse_single_range( + range_header, image_size + ) + except ValueError as e: + if "unsatisfiable" in str(e).lower(): + self._send_range_not_satisfiable(image_size) + else: self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"Content-Length ({content_length}) must equal range length ({expected_len})", + HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" ) - return - offset = start_off - remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {bytes_written} bytes", - ) - return - conn.pwrite(chunk, offset) - n = len(chunk) - offset += n - remaining -= n - bytes_written += n + return + expected_len = end_inclusive - start_off + 1 + if content_length != expected_len: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"Content-Length ({content_length}) must equal range length ({expected_len})", + ) + return + bytes_written, ok = self._write_range_nbd(image_id, cfg, start_off, content_length) + if not ok: + return self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) except Exception as e: logging.error("PATCH range error image_id=%s err=%r", image_id, e) @@ -1403,6 +1497,71 @@ def _handle_patch_range( image_id, bytes_written, dur, ) + def _handle_put_range( + self, + image_id: str, + cfg: Dict[str, Any], + content_range: str, + content_length: int, + flush: bool, + ) -> None: + """Handle PUT with Content-Range: bytes start-end/* for NBD backend.""" + lock = _get_image_lock(image_id) + # Block until we can write this image. + lock.acquire() + + if not _WRITE_SEM.acquire(blocking=False): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = _now_s() + bytes_written = 0 + try: + logging.info( + "PUT range start image_id=%s Content-Range=%s content_length=%d flush=%s", + image_id, + content_range, + content_length, + flush, + ) + try: + start_off, end_inclusive = self._parse_content_range(content_range) + except ValueError as e: + self._send_error_json( + HTTPStatus.BAD_REQUEST, f"invalid Content-Range header: {e}" + ) + return + + # Per contract we only use the start byte from Content-Range; + # length comes from Content-Length. + bytes_written, ok = self._write_range_nbd(image_id, cfg, start_off, content_length) + if not ok: + return + + if flush: + with _NbdConn(cfg["socket"], cfg.get("export")) as conn: + conn.flush() + + self._send_json( + HTTPStatus.OK, + {"ok": True, "bytes_written": bytes_written, "flushed": flush}, + ) + except Exception as e: + logging.error("PUT range error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + _WRITE_SEM.release() + lock.release() + dur = _now_s() - start + logging.info( + "PUT range end image_id=%s bytes=%d duration_s=%.3f flush=%s", + image_id, + bytes_written, + dur, + flush, + ) + def main() -> None: parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") From eac69435b1e217e34306e9fe8dcc2970af8fdeb2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 2 Mar 2026 17:03:43 +0530 Subject: [PATCH 053/129] fix put disk Signed-off-by: Abhishek Kumar --- .../veeam/api/DisksRouteHandler.java | 23 ++++++++++++++++--- .../cloudstack/veeam/api/VmsRouteHandler.java | 1 - 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 011dfe9d1b06..7ba8daf28650 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -78,8 +78,9 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path if (CollectionUtils.isNotEmpty(idAndSubPath)) { String id = idAndSubPath.get(0); if (idAndSubPath.size() == 1) { - if (!"GET".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method)) { - io.methodNotAllowed(resp, "GET, DELETE", outFormat); + if (!"GET".equalsIgnoreCase(method) && !"DELETE".equalsIgnoreCase(method) && + !"PUT".equalsIgnoreCase(method)) { + io.methodNotAllowed(resp, "GET, DELETE, PUT", outFormat); return; } if ("GET".equalsIgnoreCase(method)) { @@ -90,6 +91,10 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path handleDeleteById(id, resp, outFormat, io); return; } + if ("PUT".equalsIgnoreCase(method)) { + handlePutById(id, req, resp, outFormat, io); + return; + } } else if (idAndSubPath.size() == 2) { String subPath = idAndSubPath.get(1); if ("copy".equals(subPath)) { @@ -123,7 +128,6 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req, logger); - logger.info("Received POST request on /api/disks endpoint. Request-data: {}", data); // ToDo: remove try { Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); Disk response = serverAdapter.handleCreateDisk(request); @@ -153,6 +157,19 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, } } + protected void handlePutById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req, logger); + try { + // ToDo: do what? +// serverAdapter.deleteDisk(id); + Disk response = serverAdapter.getDisk(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (InvalidParameterValueException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } + } + protected void handlePostDiskCopy(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index eba432b7879d..dba1c2bd1695 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -333,7 +333,6 @@ protected void handleGetById(final String id, final HttpServletRequest req, fina protected void handleUpdateById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req, logger); - logger.info("Received PUT request. Request-data: {}", data); try { Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); Vm response = serverAdapter.updateInstance(id, request); From a0be1fb7721f1878d8bff9d8f1284eaddeb8aa54 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 2 Mar 2026 17:04:13 +0530 Subject: [PATCH 054/129] temp fix for orphan image transfer listing and backup removal Signed-off-by: Abhishek Kumar --- .../backup/IncrementalBackupServiceImpl.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 534ae75384f3..deca4e9a7cf4 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -176,6 +176,12 @@ public Backup createBackup(StartBackupCmd cmd) { return backupDao.persist(backup); } + protected void removedFailedBackup(BackupVO backup) { + backup.setStatus(Backup.Status.Error); + backupDao.update(backup.getId(), backup); + backupDao.remove(backup.getId()); + } + @Override public Backup startBackup(StartBackupCmd cmd) { BackupVO backup = backupDao.findById(cmd.getEntityId()); @@ -213,12 +219,14 @@ public Backup startBackup(StartBackupCmd cmd) { answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); } } catch (AgentUnavailableException | OperationTimedoutException e) { - backupDao.remove(backup.getId()); + removedFailedBackup(backup); + logger.error("Failed to communicate with agent on {} for {} start", host, backup, e); throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { - backupDao.remove(backup.getId()); + removedFailedBackup(backup); + logger.error("Failed to start {} due to: {}", backup, answer.getDetails()); throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); } @@ -710,7 +718,8 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans response.setId(imageTransferVO.getUuid()); Long backupId = imageTransferVO.getBackupId(); if (backupId != null) { - Backup backup = backupDao.findById(backupId); + // ToDo: Orphan image transfer record if backup is deleted before transfer finalization, need to clean up + Backup backup = backupDao.findByIdIncludingRemoved(backupId); response.setBackupId(backup.getUuid()); } Long volumeId = imageTransferVO.getDiskId(); From 05a5b03d9591c4f20fba28b46a163d04402c5e61 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 11 Mar 2026 12:32:05 +0530 Subject: [PATCH 055/129] changes for user assignement; refactor - make service account configurable - allow assigning vm, volume to network account Signed-off-by: Abhishek Kumar --- .../java/com/cloud/user/AccountService.java | 4 + .../api/command/admin/vm/AssignVMCmd.java | 30 ++ .../command/user/volume/AssignVolumeCmd.java | 15 + .../framework/jobs/dao/AsyncJobDao.java | 2 + .../framework/jobs/dao/AsyncJobDaoImpl.java | 10 + .../LibvirtStartBackupCommandWrapper.java | 4 +- .../cloudstack/veeam/VeeamControlService.java | 15 +- .../veeam/VeeamControlServiceImpl.java | 4 +- .../veeam/adapter/ServerAdapter.java | 347 ++++++++++++------ .../veeam/api/DisksRouteHandler.java | 2 +- .../veeam/api/ImageTransfersRouteHandler.java | 14 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 40 +- .../AsyncJobJoinVOToJobConverter.java | 6 + .../converter/UserVmJoinVOToVmConverter.java | 6 +- .../VolumeJoinVOToDiskConverter.java | 43 +-- .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 32 +- .../management/MockAccountManager.java | 31 +- .../cloud/api/query/dao/AsyncJobJoinDao.java | 4 + .../api/query/dao/AsyncJobJoinDaoImpl.java | 18 +- .../com/cloud/user/AccountManagerImpl.java | 14 + .../java/com/cloud/vm/UserVmManagerImpl.java | 5 +- .../backup/IncrementalBackupServiceImpl.java | 9 +- 22 files changed, 463 insertions(+), 192 deletions(-) diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4145e2b89eb3..f0640abf8793 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -88,10 +88,14 @@ User createUser(String userName, String password, String firstName, String lastN Account getActiveAccountById(long accountId); + Account getActiveAccountByUuid(String accountUuid); + Account getAccount(long accountId); User getActiveUser(long userId); + User getOneActiveUserForAccount(Account account); + User getUserIncludingRemoved(long userId); boolean isRootAdmin(Long accountId); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java index e11d20d06466..0e5d598505fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java @@ -85,6 +85,8 @@ public class AssignVMCmd extends BaseCmd { "In case no security groups are provided the Instance is part of the default security group.") private List securityGroupIdList; + private boolean skipNetwork = false; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -113,6 +115,34 @@ public List getSecurityGroupIdList() { return securityGroupIdList; } + public boolean isSkipNetwork() { + return skipNetwork; + } + + ///////////////////////////////////////////////////// + /////////////////// Setters ///////////////////////// + ///////////////////////////////////////////////////// + + public void setVirtualMachineId(Long virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setSkipNetwork(boolean skipNetwork) { + this.skipNetwork = skipNetwork; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java index f39853512281..f50abaf73c96 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java @@ -70,6 +70,21 @@ public Long getProjectid() { return projectid; } + ///////////////////////////////////////////////////// + /////////////////// Setter/////////////////////////// + ///////////////////////////////////////////////////// + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public void setProjectId(Long projectid) { + this.projectid = projectid; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 9f7a4ad6e058..9aba2ba97fdc 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -50,4 +50,6 @@ public interface AsyncJobDao extends GenericDao { // Returns the number of pending jobs for the given Management server msids. // NOTE: This is the msid and NOT the id long countPendingNonPseudoJobs(Long... msIds); + + List listPendingJobIdsForAccount(long accountId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index a2f1f36b8637..1dfb1738f0eb 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -266,4 +266,14 @@ public long countPendingJobs(String havingInfo, String... cmds) { List results = customSearch(sc, null); return results.get(0); } + + @Override + public List listPendingJobIdsForAccount(long accountId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.selectFields(sb.entity().getId()); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + return customSearch(sc, null); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 04416559c578..4c0087cccef0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -88,8 +88,8 @@ public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingReso script.add(backupCmd); String result = script.execute(); - backupXmlFile.delete(); - checkpointXmlFile.delete(); +// backupXmlFile.delete(); +// checkpointXmlFile.delete(); if (result != null) { return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java index adf02be8dd17..38e350d59998 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -31,8 +31,19 @@ public interface VeeamControlService extends PluggableService, Configurable { "8090", "Port for Veeam Integration REST API server", false); ConfigKey ContextPath = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.context.path", "/ovirt-engine", "Context path for Veeam Integration REST API server", false); - ConfigKey Username = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.username", + ConfigKey Username = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.api.username", "veeam", "Username for Basic Auth on Veeam Integration REST API server", true); - ConfigKey Password = new ConfigKey<>("Advanced", String.class, "integration.veeam.api.password", + ConfigKey Password = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.api.password", "change-me", "Password for Basic Auth on Veeam Integration REST API server", true); + ConfigKey ServiceAccountId = new ConfigKey<>("Advanced", String.class, + "integration.veeam.control.service.account", "", + "ID of the service account used to perform operations on resources. " + + "Preferably an admin-level account with permissions to access resources across the environment " + + "and optionally assign them to other users.", + true); + ConfigKey InstanceRestoreAssignOwner = new ConfigKey<>("Advanced", Boolean.class, + "integration.veeam.instance.restore.assign.owner", + "false", "Attempt to assign restored Instance to the owner based on OVF and network " + + "details. If the assignment fails or set to false then the Instance will remain owned by the service " + + "account", true); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java index 12e6b58b1ffd..683d0052f9d3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java @@ -76,7 +76,9 @@ public ConfigKey[] getConfigKeys() { Port, ContextPath, Username, - Password + Password, + ServiceAccountId, + InstanceRestoreAssignOwner }; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index c816ef41f31c..d83c64504f57 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; @@ -54,6 +55,8 @@ import org.apache.cloudstack.api.command.user.vm.StopVMCmd; import org.apache.cloudstack.api.command.user.vmsnapshot.CreateVMSnapshotCmd; import org.apache.cloudstack.api.command.user.vmsnapshot.DeleteVMSnapshotCmd; +import org.apache.cloudstack.api.command.user.vmsnapshot.RevertToVMSnapshotCmd; +import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; @@ -72,7 +75,6 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; -import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -116,7 +118,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import com.cloud.api.query.dao.AsyncJobJoinDao; import com.cloud.api.query.dao.DataCenterJoinDao; @@ -138,9 +139,11 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.NetworkModel; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; @@ -173,6 +176,9 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +// ToDo: fix list APIs to support pagination, etc +// ToDo: check access on objects + public class ServerAdapter extends ManagerBase { private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; private static final String SERVICE_ACCOUNT_ROLE_NAME = "Veeam Service Role"; @@ -279,7 +285,8 @@ public class ServerAdapter extends ManagerBase { @Inject ResourceTagDao resourceTagDao; - //ToDo: check access on objects + @Inject + NetworkModel networkModel; protected static Tag getDummyTagByName(String name) { Tag tag = new Tag(); @@ -340,7 +347,7 @@ protected UserAccount createServiceAccount() { } } - protected Pair createServiceAccountIfNeeded() { + protected Pair getDefaultServiceAccount() { UserAccount userAccount = accountService.getActiveUserAccount(SERVICE_ACCOUNT_NAME, 1L); if (userAccount == null) { userAccount = createServiceAccount(); @@ -351,9 +358,64 @@ protected Pair createServiceAccountIfNeeded() { accountService.getActiveAccountById(userAccount.getAccountId())); } + protected Pair getServiceAccount() { + String serviceAccountUuid = VeeamControlService.ServiceAccountId.value(); + if (StringUtils.isEmpty(serviceAccountUuid)) { + throw new CloudRuntimeException("Service account is not configured, unable to proceed"); + } + Account account = accountService.getActiveAccountByUuid(serviceAccountUuid); + if (account == null) { + throw new CloudRuntimeException("Service account with ID " + serviceAccountUuid + " not found, unable to proceed"); + } + User user = accountService.getOneActiveUserForAccount(account); + if (user == null) { + throw new CloudRuntimeException("No active user found for service account with ID " + serviceAccountUuid); + } + return new Pair<>(user, account); + } + + protected void waitForJobCompletion(long jobId) { + long timeoutNanos = TimeUnit.MINUTES.toNanos(5); + final long deadline = System.nanoTime() + timeoutNanos; + long sleepMillis = 500; + while (true) { + AsyncJobVO job = asyncJobDao.findById(jobId); + if (job == null) { + logger.warn("Async job with ID {} not found", jobId); + return; + } + if (job.getStatus() == AsyncJobVO.Status.SUCCEEDED || job.getStatus() == AsyncJobVO.Status.FAILED) { + return; + } + if (System.nanoTime() > deadline) { + logger.warn("Timed out waiting for {} completion", job); + } + try { + Thread.sleep(sleepMillis); + // back off gradually to reduce DB pressure + sleepMillis = Math.min(5000, sleepMillis + 500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Interrupted while waiting for async job completion"); + } + } + } + + protected void waitForJobCompletion(AsyncJobJoinVO job) { + if (job == null) { + logger.warn("Async job not found"); + return; + } + if (job.getStatus() == AsyncJobVO.Status.SUCCEEDED.ordinal() || + job.getStatus() == AsyncJobVO.Status.FAILED.ordinal()) { + logger.warn("Async job with ID {} already completed with status {}", job.getId(), job.getStatus()); + } + waitForJobCompletion(job.getId()); + } + @Override public boolean start() { - createServiceAccountIfNeeded(); + getServiceAccount(); //find public custom disk offering return true; } @@ -445,7 +507,6 @@ public VnicProfile getVnicProfile(String uuid) { } public List listAllInstances() { - // Todo: add filtering, pagination List vms = userVmJoinDao.listAll(); return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById); } @@ -512,7 +573,7 @@ public Vm createInstance(Vm request) { bootType = ApiConstants.BootType.UEFI; bootMode = ApiConstants.BootMode.SECURE; } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createInstance(zoneId, clusterId, name, displayName, cpu, memory, userdata, bootType, bootMode); @@ -561,7 +622,7 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, String dis if (bootMode != null) { cmd.setBootMode(bootMode.toString()); } - // ToDo: handle other. + // ToDo: handle any other field? cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); cmd.setBlankInstance(true); Map details = new HashMap<>(); @@ -581,19 +642,33 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, String dis } public Vm updateInstance(String uuid, Vm request) { - // ToDo: what to do?! + logger.warn("Received request to update VM with ID {}. No action, returning existing VM data.", uuid); return getInstance(uuid, false, false, false); } - public void deleteInstance(String uuid) { + public VmAction deleteInstance(String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + Pair serviceUserAccount = getServiceAccount(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - userVmService.destroyVm(vo.getId(), true); - } catch (ResourceUnavailableException e) { + DestroyVMCmd cmd = new DestroyVMCmd(); + cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.ID, vo.getUuid()); + params.put(ApiConstants.EXPUNGE, Boolean.TRUE.toString()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); + return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + } catch (Exception e) { throw new CloudRuntimeException("Failed to delete VM: " + e.getMessage(), e); + } finally { + CallContext.unregister(); } } @@ -602,7 +677,7 @@ public VmAction startInstance(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartVMCmd cmd = new StartVMCmd(); @@ -627,7 +702,7 @@ public VmAction stopInstance(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); @@ -653,7 +728,7 @@ public VmAction shutdownInstance(String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); @@ -674,9 +749,13 @@ public VmAction shutdownInstance(String uuid) { } } + protected Long getVolumePhysicalSize(VolumeJoinVO vo) { + return volumeApiService.getVolumePhysicalSize(vo.getFormat(), vo.getPath(), vo.getChainInfo()); + } + public List listAllDisks() { List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM); - return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes); + return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes, this::getVolumePhysicalSize); } public Disk getDisk(String uuid) { @@ -684,7 +763,7 @@ public Disk getDisk(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); } - return VolumeJoinVOToDiskConverter.toDisk(vo); + return VolumeJoinVOToDiskConverter.toDisk(vo, this::getVolumePhysicalSize); } public Disk copyDisk(String uuid) { @@ -723,7 +802,7 @@ public Disk reduceDisk(String uuid) { protected List listDiskAttachmentsByInstanceId(final long instanceId) { List kvmVolumes = volumeJoinDao.listByInstanceId(instanceId); - return VolumeJoinVOToDiskConverter.toDiskAttachmentList(kvmVolumes); + return VolumeJoinVOToDiskConverter.toDiskAttachmentList(kvmVolumes, this::getVolumePhysicalSize); } public List listDiskAttachmentsByInstanceUuid(final String uuid) { @@ -734,7 +813,35 @@ public List listDiskAttachmentsByInstanceUuid(final String uuid) return listDiskAttachmentsByInstanceId(vo.getId()); } - public DiskAttachment handleInstanceAttachDisk(final String vmUuid, final DiskAttachment request) { + protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId, Pair serviceUserAccount) { + Account account = accountService.getActiveAccountById(accountId); + if (account == null) { + throw new InvalidParameterValueException("Account with ID " + accountId + " not found"); + } + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + AssignVolumeCmd cmd = new AssignVolumeCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + cmd.setVolumeId(volumeVO.getId()); + params.put(ApiConstants.VOLUME_ID, volumeVO.getUuid()); + if (Account.Type.PROJECT.equals(account.getType())) { + cmd.setProjectId(account.getId()); + params.put(ApiConstants.PROJECT_ID, account.getUuid()); + } else { + cmd.setAccountId(account.getId()); + params.put(ApiConstants.ACCOUNT_ID, account.getUuid()); + } + cmd.setFullUrlParams(params); + volumeApiService.assignVolumeToAccount(cmd); + } catch (ResourceAllocationException | CloudRuntimeException e) { + logger.error("Failed to assign {} to {}: {}", volumeVO, account, e.getMessage(), e); + } finally { + CallContext.unregister(); + } + } + + public DiskAttachment attachInstanceDisk(final String vmUuid, final DiskAttachment request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); @@ -746,12 +853,25 @@ public DiskAttachment handleInstanceAttachDisk(final String vmUuid, final DiskAt if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); + if (vmVo.getAccountId() != volumeVO.getAccountId()) { + if (VeeamControlService.InstanceRestoreAssignOwner.value()) { + assignVolumeToAccount(volumeVO, vmVo.getAccountId(), serviceUserAccount); + } else { + throw new PermissionDeniedException("Disk with ID " + request.getDisk().getId() + + " belongs to a different account and cannot be attached to the VM"); + } + } CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), 0L, false); + Long deviceId = null; + List volumes = volumeDao.findUsableVolumesForInstance(vmVo.getId()); + if (CollectionUtils.isEmpty(volumes)) { + deviceId = 0L; + } + Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), deviceId, false); VolumeJoinVO attachedVolumeVO = volumeJoinDao.findById(volume.getId()); - return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO); + return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO, this::getVolumePhysicalSize); } finally { CallContext.unregister(); } @@ -765,7 +885,7 @@ public void deleteDisk(String uuid) { volumeApiService.deleteVolume(vo.getId(), accountService.getSystemAccount()); } - public Disk handleCreateDisk(Disk request) { + public Disk createDisk(Disk request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); } @@ -805,7 +925,7 @@ public Disk handleCreateDisk(Disk request) { initialSize = Long.parseLong(request.getInitialSize()); } catch (NumberFormatException ignored) {} } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); Account serviceAccount = serviceUserAccount.second(); DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { @@ -815,7 +935,7 @@ public Disk handleCreateDisk(Disk request) { if (diskOfferingId == null) { throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); } - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); } finally { @@ -841,7 +961,7 @@ private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, } // Implementation for creating a Disk resource - return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId())); + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId()), this::getVolumePhysicalSize); } protected List listNicsByInstance(final long instanceId, final String instanceUuid) { @@ -861,7 +981,42 @@ public List listNicsByInstanceUuid(final String uuid) { return listNicsByInstance(vo.getId(), vo.getUuid()); } - public Nic handleAttachInstanceNic(final String vmUuid, final Nic request) { + protected boolean accountCannotAccessNetwork(NetworkVO networkVO, long accountId) { + Account account = accountService.getActiveAccountById(accountId); + try { + networkModel.checkNetworkPermissions(account, networkVO); + return false; + } catch (CloudRuntimeException e) { + logger.debug("{} cannot access {}: {}", account, networkVO, e.getMessage()); + } + return true; + } + + protected void assignVmToAccount(UserVmVO vmVO, long accountId, Pair serviceUserAccount) { + Account account = accountService.getActiveAccountById(accountId); + if (account == null) { + throw new InvalidParameterValueException("Account with ID " + accountId + " not found"); + } + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + AssignVMCmd cmd = new AssignVMCmd(); + ComponentContext.inject(cmd); + cmd.setVirtualMachineId(vmVO.getId()); + cmd.setAccountName(account.getAccountName()); + cmd.setDomainId(account.getDomainId()); + if (Account.Type.PROJECT.equals(account.getType())) { + cmd.setProjectId(account.getId()); + } + userVmService.moveVmToUser(cmd); + } catch (ResourceAllocationException | CloudRuntimeException | ResourceUnavailableException | + InsufficientCapacityException e) { + logger.error("Failed to assign {} to {}: {}", vmVO, account, e.getMessage(), e); + } finally { + CallContext.unregister(); + } + } + + public Nic attachInstanceNic(final String vmUuid, final Nic request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); @@ -873,7 +1028,13 @@ public Nic handleAttachInstanceNic(final String vmUuid, final Nic request) { if (networkVO == null) { throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().getId() + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); + if (vmVo.getAccountId() != networkVO.getAccountId() && + networkVO.getAccountId() != Account.ACCOUNT_ID_SYSTEM && + VeeamControlService.InstanceRestoreAssignOwner.value() && + accountCannotAccessNetwork(networkVO, vmVo.getAccountId())) { + assignVmToAccount(vmVo, networkVO.getAccountId(), serviceUserAccount); + } CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { AddNicToVMCmd cmd = new AddNicToVMCmd(); @@ -907,7 +1068,7 @@ public ImageTransfer getImageTransfer(String uuid) { return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); } - public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { + public ImageTransfer createImageTransfer(ImageTransfer request) { if (request == null) { throw new InvalidParameterValueException("Request image transfer data is empty"); } @@ -934,7 +1095,7 @@ public ImageTransfer handleCreateImageTransfer(ImageTransfer request) { return createImageTransfer(backupId, volumeVO.getId(), direction, format); } - public boolean handleCancelImageTransfer(String uuid) { + public boolean cancelImageTransfer(String uuid) { ImageTransferVO vo = imageTransferDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); @@ -942,7 +1103,7 @@ public boolean handleCancelImageTransfer(String uuid) { return incrementalBackupService.cancelImageTransfer(vo.getId()); } - public boolean handleFinalizeImageTransfer(String uuid) { + public boolean finalizeImageTransfer(String uuid) { ImageTransferVO vo = imageTransferDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); @@ -951,7 +1112,7 @@ public boolean handleFinalizeImageTransfer(String uuid) { } private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = @@ -992,8 +1153,10 @@ protected NetworkVO getNetworkById(Long networkId) { } public List listAllJobs() { - // ToDo: find active jobs for service account - return Collections.emptyList(); + Pair serviceUserAccount = getServiceAccount(); + List jobIds = asyncJobDao.listPendingJobIdsForAccount(serviceUserAccount.second().getId()); + List jobJoinVOs = asyncJobJoinDao.listByIds(jobIds); + return AsyncJobJoinVOToJobConverter.toJobList(jobJoinVOs); } public Job getJob(String uuid) { @@ -1013,12 +1176,12 @@ public List listSnapshotsByInstanceUuid(final String uuid) { return VmSnapshotVOToSnapshotConverter.toSnapshotList(snapshots, vo.getUuid()); } - public Snapshot handleCreateInstanceSnapshot(final String vmUuid, final Snapshot request) { + public Snapshot createInstanceSnapshot(final String vmUuid, final Snapshot request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { CreateVMSnapshotCmd cmd = new CreateVMSnapshotCmd(); @@ -1060,7 +1223,7 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); @@ -1074,10 +1237,10 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for snapshot deletion"); } - action = AsyncJobJoinVOToJobConverter.toAction(jobVo); - if (async) { - // ToDo: wait for job completion? + if (!async) { + waitForJobCompletion(jobVo); } + action = AsyncJobJoinVOToJobConverter.toAction(jobVo); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete snapshot: " + e.getMessage(), e); } finally { @@ -1086,34 +1249,33 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { return action; } - public ResourceAction revertToSnapshot(String uuid) { - throw new InvalidParameterValueException("revertToSnapshot with ID " + uuid + " not implemented"); -// ResourceAction action = null; -// VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); -// } -// Pair serviceUserAccount = createServiceAccountIfNeeded(); -// CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); -// try { -// RevertToVMSnapshotCmd cmd = new RevertToVMSnapshotCmd(); -// ComponentContext.inject(cmd); -// Map params = new HashMap<>(); -// params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); -// ApiServerService.AsyncCmdResult result = -// apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), -// serviceUserAccount.second()); -// AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); -// if (jobVo == null) { -// throw new CloudRuntimeException("Failed to find job for snapshot revert"); -// } -// action = AsyncJobJoinVOToJobConverter.toAction(jobVo); -// } catch (Exception e) { -// throw new CloudRuntimeException("Failed to revert to snapshot: " + e.getMessage(), e); -// } finally { -// CallContext.unregister(); -// } -// return action; + public ResourceAction revertInstanceToSnapshot(String uuid) { + ResourceAction action = null; + VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); + } + Pair serviceUserAccount = getServiceAccount(); + CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + RevertToVMSnapshotCmd cmd = new RevertToVMSnapshotCmd(); + ComponentContext.inject(cmd); + Map params = new HashMap<>(); + params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); + ApiServerService.AsyncCmdResult result = + apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), + serviceUserAccount.second()); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for snapshot revert"); + } + action = AsyncJobJoinVOToJobConverter.toAction(jobVo); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to revert to snapshot: " + e.getMessage(), e); + } finally { + CallContext.unregister(); + } + return action; } public List listBackupsByInstanceUuid(final String uuid) { @@ -1130,7 +1292,7 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartBackupCmd cmd = new StartBackupCmd(); @@ -1156,42 +1318,6 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { } } - @Nullable - private BackupVO getBackupFromJob(ApiServerService.AsyncCmdResult result, UserVmVO vmVo) { - AsyncJobVO jobVo = null; - // wait for job to complete and get backup ID - long timeoutNanos = TimeUnit.MINUTES.toNanos(2); - final long deadline = System.nanoTime() + timeoutNanos; - long sleepMillis = 1000; - while (System.nanoTime() < deadline) { - jobVo = asyncJobDao.findByIdIncludingRemoved(result.jobId); - if (jobVo == null) { - throw new CloudRuntimeException("Failed to find job for backup creation"); - } - if (!JobInfo.Status.IN_PROGRESS.equals(jobVo.getStatus())) { - break; - } - try { - Thread.sleep(sleepMillis); - // back off gradually to reduce DB pressure - sleepMillis = Math.min(5000, sleepMillis + 500); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new CloudRuntimeException("Interrupted while waiting for backup creation job", ie); - } - } - // if still in progress after timeout, fail fast - if (jobVo != null && JobInfo.Status.IN_PROGRESS.equals(jobVo.getStatus())) { - throw new CloudRuntimeException("Timed out waiting for backup creation job"); - } - BackupVO vo = null; - List backups = backupDao.searchByVmIds(List.of(vmVo.getId())); - if (CollectionUtils.isNotEmpty(backups)) { - vo = backups.get(0); - } - return vo; - } - public Backup getBackup(String uuid) { BackupVO vo = backupDao.findByUuidIncludingRemoved(uuid); if (vo == null) { @@ -1203,12 +1329,7 @@ public Backup getBackup(String uuid) { public List listDisksByBackupUuid(final String uuid) { throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implemented"); -// ToDo: implement -// BackupVO vo = backupDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); -// } -// return VolumeJoinVOToDiskConverter.toDiskList(volumes); + // This won't be feasible with current structure } public Backup finalizeBackup(final String vmUuid, final String backupUuid) { @@ -1220,7 +1341,7 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { if (backup == null) { throw new InvalidParameterValueException("Backup with ID " + backupUuid + " not found"); } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { FinalizeBackupCmd cmd = new FinalizeBackupCmd(); @@ -1271,7 +1392,7 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { logger.warn("Checkpoint ID {} does not match active checkpoint for VM {}", checkpointId, vmUuid); return; } - Pair serviceUserAccount = createServiceAccountIfNeeded(); + Pair serviceUserAccount = getServiceAccount(); CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { DeleteVmCheckpointCmd cmd = new DeleteVmCheckpointCmd(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 7ba8daf28650..f0fc1368d56d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -130,7 +130,7 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons String data = RouteHandler.getRequestData(req, logger); try { Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); - Disk response = serverAdapter.handleCreateDisk(request); + Disk response = serverAdapter.createDisk(request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 6a26d54beaf7..33371bc3c354 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -115,7 +115,7 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons String data = RouteHandler.getRequestData(req, logger); try { ImageTransfer request = io.getMapper().jsonMapper().readValue(data, ImageTransfer.class); - ImageTransfer response = serverAdapter.handleCreateImageTransfer(request); + ImageTransfer response = serverAdapter.createImageTransfer(request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); @@ -128,27 +128,27 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi ImageTransfer response = serverAdapter.getImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } protected void handleCancelById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - serverAdapter.handleCancelImageTransfer(id); + serverAdapter.cancelImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer cancelled successfully", outFormat); - } catch (InvalidParameterValueException e) { - io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } protected void handleFinalizeById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - serverAdapter.handleFinalizeImageTransfer(id); + serverAdapter.finalizeImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Image transfer finalized successfully", outFormat); } catch (CloudRuntimeException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index dba1c2bd1695..22c8286878dc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -345,15 +345,15 @@ protected void handleUpdateById(final String id, final HttpServletRequest req, f protected void handleDeleteById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { - serverAdapter.deleteInstance(id); - io.getWriter().write(resp, HttpServletResponse.SC_OK, "", outFormat); + VmAction vm = serverAdapter.deleteInstance(id); + io.getWriter().write(resp, HttpServletResponse.SC_OK, vm, outFormat); } catch (CloudRuntimeException e) { io.notFound(resp, e.getMessage(), outFormat); } } - protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { VmAction vm = serverAdapter.startInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -362,8 +362,8 @@ protected void handleStartVmById(final String id, final HttpServletRequest req, } } - protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { VmAction vm = serverAdapter.stopInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -372,8 +372,8 @@ protected void handleStopVmById(final String id, final HttpServletRequest req, f } } - protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { VmAction vm = serverAdapter.shutdownInstance(id); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -382,8 +382,8 @@ protected void handleShutdownVmById(final String id, final HttpServletRequest re } } - protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { List disks = serverAdapter.listDiskAttachmentsByInstanceUuid(id); NamedList response = NamedList.of("disk_attachment", disks); @@ -399,15 +399,15 @@ protected void handlePostDiskAttachmentForVmId(final String id, final HttpServle String data = RouteHandler.getRequestData(req, logger); try { DiskAttachment request = io.getMapper().jsonMapper().readValue(data, DiskAttachment.class); - DiskAttachment response = serverAdapter.handleInstanceAttachDisk(id, request); + DiskAttachment response = serverAdapter.attachInstanceDisk(id, request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } } - protected void handleGetNicsByVmId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleGetNicsByVmId(final String id, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { List nics = serverAdapter.listNicsByInstanceUuid(id); NamedList response = NamedList.of("nic", nics); @@ -423,7 +423,7 @@ protected void handlePostNicForVmId(final String id, final HttpServletRequest re String data = RouteHandler.getRequestData(req, logger); try { Nic request = io.getMapper().jsonMapper().readValue(data, Nic.class); - Nic response = serverAdapter.handleAttachInstanceNic(id, request); + Nic response = serverAdapter.attachInstanceNic(id, request); io.getWriter().write(resp, HttpServletResponse.SC_CREATED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); @@ -447,7 +447,7 @@ protected void handlePostSnapshotForVmId(final String id, final HttpServletReque String data = RouteHandler.getRequestData(req, logger); try { Snapshot request = io.getMapper().jsonMapper().readValue(data, Snapshot.class); - Snapshot response = serverAdapter.handleCreateInstanceSnapshot(id, request); + Snapshot response = serverAdapter.createInstanceSnapshot(id, request); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); } catch (JsonProcessingException | CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); @@ -455,7 +455,7 @@ protected void handlePostSnapshotForVmId(final String id, final HttpServletReque } protected void handleGetSnapshotById(final String id, final HttpServletResponse resp, - final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { try { Snapshot response = serverAdapter.getSnapshot(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); @@ -484,9 +484,13 @@ protected void handleDeleteSnapshotById(final String id, final HttpServletReques protected void handleRestoreSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - //ToDo: implement String data = RouteHandler.getRequestData(req, logger); - io.badRequest(resp, "Not implemented", outFormat); + try { + ResourceAction response = serverAdapter.revertInstanceToSnapshot(id); + io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetBackupsByVmId(final String id, final HttpServletResponse resp, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index 49bf1f1cabad..625c9d9e469a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -18,6 +18,8 @@ package org.apache.cloudstack.veeam.api.converter; import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.veeam.VeeamControlService; @@ -83,6 +85,10 @@ public static Job toJob(AsyncJobJoinVO vo) { return job; } + public static List toJobList(List vos) { + return vos.stream().map(AsyncJobJoinVOToJobConverter::toJob).collect(Collectors.toList()); + } + protected static void fillAction(final ResourceAction action, final AsyncJobJoinVO vo) { final String basePath = VeeamControlService.ContextPath.value(); action.setJob(Ref.of(basePath + JobsRouteHandler.BASE_ROUTE + vo.getUuid(), vo.getUuid())); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 6c7c8bddd794..42431dc357b3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -160,16 +160,16 @@ public static Vm toVm(final UserVmJoinVO src, final Function h basePath + ApiService.BASE_ROUTE + "/cpuprofiles/" + src.getServiceOfferingUuid(), src.getServiceOfferingUuid())); if (allContent) { - dst.setInitialization(getOvfInitialization(dst)); + dst.setInitialization(getOvfInitialization(dst, src)); } return dst; } - private static Vm.Initialization getOvfInitialization(Vm vm) { + private static Vm.Initialization getOvfInitialization(Vm vm, UserVmJoinVO vo) { final Vm.Initialization.Configuration configuration = new Vm.Initialization.Configuration(); configuration.setType("ovf"); - configuration.setData(OvfXmlUtil.toXml(vm)); + configuration.setData(OvfXmlUtil.toXml(vm, vo)); final Vm.Initialization initialization = new Vm.Initialization(); initialization.setConfiguration(configuration); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index 497f4d7f441b..b1be9b988042 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.cloudstack.backup.Backup; @@ -34,14 +35,12 @@ import org.apache.cloudstack.veeam.api.dto.StorageDomain; import org.apache.cloudstack.veeam.api.dto.Vm; -import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.VolumeJoinVO; import com.cloud.storage.Storage; import com.cloud.storage.Volume; -import com.cloud.storage.VolumeStats; public class VolumeJoinVOToDiskConverter { - public static Disk toDisk(final VolumeJoinVO vol) { + public static Disk toDisk(final VolumeJoinVO vol, final Function physicalSizeResolver) { final Disk disk = new Disk(); final String basePath = VeeamControlService.ContextPath.value(); final String apiBasePath = basePath + ApiService.BASE_ROUTE; @@ -64,19 +63,12 @@ public static Disk toDisk(final VolumeJoinVO vol) { disk.setProvisionedSize(String.valueOf(size)); disk.setActualSize(String.valueOf(actualSize)); disk.setTotalSize(String.valueOf(size)); - VolumeStats vs = null; - if (List.of(Storage.ImageFormat.VHD, Storage.ImageFormat.QCOW2, Storage.ImageFormat.RAW).contains(vol.getFormat())) { - if (vol.getPath() != null) { - vs = ApiDBUtils.getVolumeStatistics(vol.getPath()); - } - } else if (vol.getFormat() == Storage.ImageFormat.OVA) { - if (vol.getChainInfo() != null) { - vs = ApiDBUtils.getVolumeStatistics(vol.getChainInfo()); - } + Long physicalSize = null; + if (physicalSizeResolver != null) { + physicalSize = physicalSizeResolver.apply(vol); } - if (vs != null) { - disk.setTotalSize(String.valueOf(vs.getVirtualSize())); - disk.setActualSize(String.valueOf(vs.getPhysicalSize())); + if (physicalSize != null) { + disk.setActualSize(String.valueOf(physicalSize)); } // Disk format @@ -122,9 +114,10 @@ public static Disk toDisk(final VolumeJoinVO vol) { return disk; } - public static List toDiskList(final List srcList) { + public static List toDiskList(final List srcList, + final Function physicalSizeResolver) { return srcList.stream() - .map(VolumeJoinVOToDiskConverter::toDisk) + .map(vo -> toDisk(vo, physicalSizeResolver)) .collect(Collectors.toList()); } @@ -143,7 +136,8 @@ public static List toDiskListFromVolumeInfos(final List return disks; } - public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { + public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol, + final Function physicalSizeResolver) { final DiskAttachment da = new DiskAttachment(); final String basePath = VeeamControlService.ContextPath.value(); @@ -154,7 +148,7 @@ public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { da.setHref(da.getVm().getHref() + "/diskattachments/" + diskAttachmentId);; // Links - da.setDisk(toDisk(vol)); + da.setDisk(toDisk(vol, physicalSizeResolver)); // Properties da.setActive("true"); @@ -167,9 +161,10 @@ public static DiskAttachment toDiskAttachment(final VolumeJoinVO vol) { return da; } - public static List toDiskAttachmentList(final List srcList) { + public static List toDiskAttachmentList(final List srcList, + final Function physicalSizeResolver) { return srcList.stream() - .map(VolumeJoinVOToDiskConverter::toDiskAttachment) + .map(vo -> toDiskAttachment(vo, physicalSizeResolver)) .collect(Collectors.toList()); } @@ -190,9 +185,9 @@ private static String mapStatus(final Volume.State state) { if (state == null) { return "ok"; } - switch (state.name().toLowerCase()) { - case "ready": - case "allocated": + switch (state) { + case Ready: + case Allocated: return "ok"; default: return "locked"; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java index b4bc8517a800..ebee1e242d65 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -42,6 +42,8 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import com.cloud.api.query.vo.UserVmJoinVO; + public class OvfXmlUtil { private static final String NS_OVF = "http://schemas.dmtf.org/ovf/envelope/1/"; @@ -58,7 +60,7 @@ public class OvfXmlUtil { return sdf; }); - public static String toXml(final Vm vm) { + public static String toXml(final Vm vm, final UserVmJoinVO vo) { final String vmId = vm.getId(); final String vmName = vm.getName(); final String vmDesc = defaultString(vm.getDescription()); @@ -169,6 +171,32 @@ public static String toXml(final Vm vm) { } sb.append(""); + if (vo != null) { + // -- Add a section for CloudStack-specific metadata that some consumers might look for (e.g. for import back into CloudStack) --- + // Add CloudStack-specific metadata section + sb.append("
"); + sb.append("CloudStack specific metadata"); + sb.append(""); + sb.append("").append(vo.getAccountUuid()).append(""); + sb.append("").append(vo.getDomainUuid()).append(""); + sb.append("").append(escapeText(vo.getProjectUuid())).append(""); + sb.append("").append(vo.getServiceOfferingUuid()).append(""); + sb.append(""); + for (DiskAttachment da : diskAttachments(vm)) { + if (da == null || da.getDisk() == null || StringUtils.isBlank(da.getDisk().getId())) { + continue; + } + final org.apache.cloudstack.veeam.api.dto.Disk d = da.getDisk(); + sb.append(""); + sb.append("").append(escapeText(d.getId())).append(""); + sb.append("").append(d.getDiskProfile().getId()).append(""); + sb.append(""); + } + sb.append(""); + sb.append(""); + sb.append("
"); + } + // --- Content / VirtualSystem --- sb.append(""); sb.append("").append(escapeText(vmName)).append(""); @@ -191,7 +219,7 @@ public static String toXml(final Vm vm) { sb.append("false"); sb.append("false"); sb.append("0"); - sb.append("").append(ZERO_UUID).append(""); + sb.append("").append(vo.getAccountUuid()).append(""); sb.append("0"); sb.append("").append(escapeText(booleanString(vm.getBios() != null && vm.getBios().getBootMenu() != null ? vm.getBios().getBootMenu().getEnabled() : null, "false"))).append(""); sb.append("true"); diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java index d9f4963165ec..ab7662f44309 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java @@ -17,18 +17,22 @@ package org.apache.cloudstack.network.contrail.management; +import java.net.InetAddress; import java.util.List; import java.util.Map; -import java.net.InetAddress; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd; +import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.RolePermissionEntity; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.acl.apikeypair.ApiKeyPair; import org.apache.cloudstack.acl.apikeypair.ApiKeyPairPermission; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.admin.account.CreateAccountCmd; +import org.apache.cloudstack.api.command.admin.account.UpdateAccountCmd; import org.apache.cloudstack.api.command.admin.user.DeleteUserCmd; import org.apache.cloudstack.api.command.admin.user.DeleteUserKeysCmd; import org.apache.cloudstack.api.command.admin.user.GetUserKeysCmd; @@ -37,20 +41,15 @@ import org.apache.cloudstack.api.command.admin.user.MoveUserCmd; import org.apache.cloudstack.api.command.admin.user.RegisterUserKeysCmd; import org.apache.cloudstack.api.command.admin.user.UpdateUserCmd; +import org.apache.cloudstack.api.response.ApiKeyPairResponse; +import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserTwoFactorAuthenticationSetupResponse; import org.apache.cloudstack.auth.UserTwoFactorAuthenticator; import org.apache.cloudstack.backup.BackupOffering; -import org.apache.cloudstack.framework.config.ConfigKey; - -import org.apache.cloudstack.acl.apikeypair.ApiKeyPair; -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.api.response.ApiKeyPairResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.command.admin.account.UpdateAccountCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd; import com.cloud.api.query.vo.ControlledViewEntity; import com.cloud.configuration.ResourceLimit; import com.cloud.configuration.dao.ResourceCountDao; @@ -614,4 +613,14 @@ public void verifyCallerPrivilegeForUserOrAccountOperations(User user) { @Override public void checkCallerRoleTypeAllowedForUserOrAccountOperations(Account userAccount, User user) { } + + @Override + public Account getActiveAccountByUuid(String accountUuid) { + return null; + } + + @Override + public User getOneActiveUserForAccount(Account account) { + return null; + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDao.java index 756425f5093e..43974bcf9cc6 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDao.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api.query.dao; +import java.util.List; + import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.framework.jobs.AsyncJob; @@ -28,4 +30,6 @@ public interface AsyncJobJoinDao extends GenericDao { AsyncJobJoinVO newAsyncJobView(AsyncJob vol); + List listByIds(List ids); + } diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java index 10ef67bbbea1..93af9a04e144 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java @@ -16,17 +16,17 @@ // under the License. package com.cloud.api.query.dao; +import java.util.Collections; import java.util.Date; import java.util.List; - import javax.inject.Inject; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; import com.cloud.api.ApiResponseHelper; import com.cloud.api.ApiSerializerHelper; @@ -115,4 +115,16 @@ public AsyncJobJoinVO newAsyncJobView(AsyncJob job) { } + @Override + public List listByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 2011d4556465..c9f4feea8e90 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -2755,6 +2755,11 @@ public Account getActiveAccountById(long accountId) { return _accountDao.findById(accountId); } + @Override + public Account getActiveAccountByUuid(String accountUuid) { + return _accountDao.findByUuid(accountUuid); + } + @Override public Account getAccount(long accountId) { return _accountDao.findByIdIncludingRemoved(accountId); @@ -2773,6 +2778,15 @@ public User getActiveUser(long userId) { return _userDao.findById(userId); } + @Override + public User getOneActiveUserForAccount(Account account) { + List users = _userDao.listByAccount(account.getId()); + if (CollectionUtils.isEmpty(users)) { + return null; + } + return users.get(0); + } + @Override public User getUserIncludingRemoved(long userId) { return _userDao.findByIdIncludingRemoved(userId); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3fa6cd105c9b..5a5f127d050e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8017,7 +8017,10 @@ public UserVm moveVmToUser(final AssignVMCmd cmd) throws ResourceAllocationExcep logger.trace("Verifying if the new account [{}] has access to the specified domain [{}].", newAccount, domain); _accountMgr.checkAccess(newAccount, domain); - Network newNetwork = ensureDestinationNetwork(cmd, vm, newAccount); + Network newNetwork = null; + if (!cmd.isSkipNetwork()) { + newNetwork = ensureDestinationNetwork(cmd, vm, newAccount); + } try { Transaction.execute(new TransactionCallbackNoReturn() { @Override diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index deca4e9a7cf4..855a9cfcb5be 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -158,7 +158,7 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setStatus(Backup.Status.ReadyForTransfer); + backup.setStatus(Backup.Status.Queued); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setDate(new Date()); @@ -236,6 +236,7 @@ public Backup startBackup(StartBackupCmd cmd) { // todo: set it in the backend backup.setType("Incremental"); } + updateBackupState(backup, Backup.Status.ReadyForTransfer); return backup; } @@ -308,12 +309,12 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { // Delete old checkpoint if exists (POC: skip actual libvirt call) if (oldCheckpointId != null) { // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete - logger.debug("Would delete old checkpoint: " + oldCheckpointId); + logger.debug("Would delete old checkpoint: {}", oldCheckpointId); } // Delete backup session record - backup.setStatus(Backup.Status.BackedUp); - backupDao.update(backupId, backup); + updateBackupState(backup, Backup.Status.BackedUp); + backupDao.remove(backup.getId()); return backup; From 10ad7967cdfa2f19cd8f34ce53c3ff267055d771 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sat, 14 Mar 2026 20:09:25 +0530 Subject: [PATCH 056/129] bug fixes --- .../backup/StartNBDServerCommand.java | 8 +++++++- .../META-INF/db/schema-42100to42200.sql | 1 - ...ibvirtCreateImageTransferCommandWrapper.java | 2 +- .../LibvirtStartBackupCommandWrapper.java | 4 ++-- .../LibvirtStartNBDServerCommandWrapper.java | 3 ++- .../LibvirtStopNBDServerCommandWrapper.java | 16 ++-------------- .../backup/IncrementalBackupServiceImpl.java | 17 +++++++++-------- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java index 47dd2b4a6df9..67a858af7f00 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -25,16 +25,18 @@ public class StartNBDServerCommand extends Command { private String volumePath; private String socket; private String direction; + private String fromCheckpointId; public StartNBDServerCommand() { } - protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction) { + protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction, String fromCheckpointId) { this.transferId = transferId; this.socket = socket; this.exportName = exportName; this.volumePath = volumePath; this.direction = direction; + this.fromCheckpointId = fromCheckpointId; } public String getExportName() { @@ -61,4 +63,8 @@ public String getVolumePath() { public String getDirection() { return direction; } + + public String getFromCheckpointId() { + return fromCheckpointId; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 044f7475324f..fbb2fd079f9c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -18,7 +18,6 @@ --; -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; -not supported for download -- health check status as enum CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index d3eca1aeb23f..db0918f5c072 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -134,7 +134,7 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r payload.put("export", exportName); String checkpointId = cmd.getCheckpointId(); if (checkpointId != null) { - payload.put("export_bitmap", exportName + "-" + checkpointId.substring(0, 4)); + payload.put("export_bitmap", cmd.getCheckpointId()); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 4c0087cccef0..04416559c578 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -88,8 +88,8 @@ public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingReso script.add(backupCmd); String result = script.execute(); -// backupXmlFile.delete(); -// checkpointXmlFile.delete(); + backupXmlFile.delete(); + checkpointXmlFile.delete(); if (result != null) { return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index 263e5f1cae5f..fb532fd2a9af 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -69,10 +69,11 @@ public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resour String socketName = "/tmp/imagetransfer/" + socket + ".sock"; String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --socket %s --persistent %s %s", + "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --socket %s --persistent %s %s %s", unitName, exportName, socketName, + cmd.getFromCheckpointId() != null ? "-B " + cmd.getFromCheckpointId() : "", cmd.getDirection().equals("download") ? "--read-only" : "", volumePath ); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java index a2de9b56d033..57c7ebb706bc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java @@ -38,7 +38,8 @@ private void resetService(String unitName) { resetScript.execute(); } - private Answer handleUpload(StopNBDServerCommand cmd) { + @Override + public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { try { String unitName = "qemu-nbd-" + cmd.getTransferId().hashCode(); @@ -68,17 +69,4 @@ private Answer handleUpload(StopNBDServerCommand cmd) { return new Answer(cmd, false, "Error finalizing image transfer: " + e.getMessage()); } } - - private Answer handleDownload(StopNBDServerCommand cmd) { - return new Answer(cmd, true, "Image transfer finalized"); - } - - @Override - public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { - if (cmd.getDirection().equals("download")) { - return handleDownload(cmd); - } else { - return handleUpload(cmd); - } - } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 855a9cfcb5be..dd4dc7565955 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -337,7 +337,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); if (vm.getState() == State.Stopped) { String volumePath = getVolumePathForFileBasedBackend(volume); - startNBDServer(transferId, direction, backup.getHostId(), volume.getUuid(), volumePath); + startNBDServer(transferId, direction, backup.getHostId(), volume.getUuid(), volumePath, vm.getActiveCheckpointId()); socket = transferId; } @@ -393,7 +393,7 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { return hosts.get(0); } - private void startNBDServer(String transferId, String direction, Long hostId, String exportName, String volumePath) { + private void startNBDServer(String transferId, String direction, Long hostId, String exportName, String volumePath, String checkpointId) { StartNBDServerAnswer nbdServerAnswer; if (hostId == null) { throw new CloudRuntimeException("Host cannot be determined for starting NBD server"); @@ -407,7 +407,8 @@ private void startNBDServer(String transferId, String direction, Long hostId, St exportName, volumePath, transferId, - direction + direction, + checkpointId ); try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(hostId, nbdServerCmd); @@ -457,7 +458,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer volumePath); } else { - startNBDServer(transferId, direction, host.getId(), volume.getUuid(), volumePath); + startNBDServer(transferId, direction, host.getId(), volume.getUuid(), volumePath, null); imageTransfer = new ImageTransferVO( transferId, null, @@ -486,7 +487,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer if (!transferAnswer.getResult()) { if (!backend.equals(ImageTransfer.Backend.file)) { - stopNbdServer(imageTransfer); + stopNBDServer(imageTransfer); } throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); } @@ -578,14 +579,14 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); if (vm.getState() == State.Stopped) { - boolean stopNbdServerResult = stopNbdServer(imageTransfer); + boolean stopNbdServerResult = stopNBDServer(imageTransfer); if (!stopNbdServerResult) { throw new CloudRuntimeException("Failed to stop the nbd server"); } } } - private boolean stopNbdServer(ImageTransferVO imageTransfer) { + private boolean stopNBDServer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); String direction = imageTransfer.getDirection().toString(); StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction); @@ -602,7 +603,7 @@ private boolean stopNbdServer(ImageTransferVO imageTransfer) { private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { String transferId = imageTransfer.getUuid(); - boolean stopNbdServerResult = stopNbdServer(imageTransfer); + boolean stopNbdServerResult = stopNBDServer(imageTransfer); if (!stopNbdServerResult) { throw new CloudRuntimeException("Failed to stop the nbd server"); } From 9974e487690fda44f9b74608ff26fa34598299d7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Mar 2026 11:16:07 +0530 Subject: [PATCH 057/129] changes for retrieving vm account from ovf Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 61 +- .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 151 +++-- .../apache/cloudstack/veeam/api/dto/Vm.java | 13 + .../src/main/resources/test.xml | 560 ++---------------- 4 files changed, 234 insertions(+), 551 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index d83c64504f57..530edf4cfa90 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -148,6 +148,8 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; +import com.cloud.projects.Project; +import com.cloud.projects.ProjectService; import com.cloud.server.ResourceTag; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Volume; @@ -164,6 +166,7 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -288,6 +291,9 @@ public class ServerAdapter extends ManagerBase { @Inject NetworkModel networkModel; + @Inject + ProjectService projectService; + protected static Tag getDummyTagByName(String name) { Tag tag = new Tag(); String id = UUID.nameUUIDFromBytes(String.format("veeam:%s", name.toLowerCase()).getBytes()).toString(); @@ -522,6 +528,28 @@ public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, bo allContent); } + Ternary getVmOwner(Vm request) { + String accountUuid = request.getAccountId(); + if (StringUtils.isBlank(accountUuid)) { + return new Ternary<>(null, null, null); + } + Account account = accountService.getActiveAccountByUuid(accountUuid); + if (account == null) { + logger.warn("Account with ID {} not found, unable to determine owner for VM creation request", accountUuid); + return new Ternary<>(null, null, null); + } + Long projectId = null; + if (Account.Type.PROJECT.equals(account.getType())) { + Project project = projectService.findByProjectAccountId(account.getId()); + if (project == null) { + logger.warn("Project for {} not found, unable to determine owner for VM creation request", account); + return new Ternary<>(null, null, null); + } + projectId = project.getId(); + } + return new Ternary<>(account.getDomainId(), account.getAccountName(), projectId); + } + public Vm createInstance(Vm request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); @@ -573,16 +601,29 @@ public Vm createInstance(Vm request) { bootType = ApiConstants.BootType.UEFI; bootMode = ApiConstants.BootMode.SECURE; } + Ternary owner = getVmOwner(request); + String serviceOfferingUuid = null; + if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { + serviceOfferingUuid = request.getCpuProfile().getId(); + } Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - return createInstance(zoneId, clusterId, name, displayName, cpu, memory, userdata, bootType, bootMode); + return createInstance(zoneId, clusterId, owner.first(), owner.second(), owner.third(), name, displayName, + serviceOfferingUuid, cpu, memory, userdata, bootType, bootMode); } finally { CallContext.unregister(); } } - protected ServiceOffering getServiceOfferingIdForVmCreation(long zoneId, int cpu, long memory) { + protected ServiceOffering getServiceOfferingIdForVmCreation(String serviceOfferingUuid, long zoneId, int cpu, long memory) { + if (StringUtils.isNotBlank(serviceOfferingUuid)) { + ServiceOffering offering = serviceOfferingDao.findByUuid(serviceOfferingUuid); + if (offering != null && !offering.isCustomized()) { + // ToDo: check offering is available in the specified zone and matches the requested cpu/memory if it's not a custom offering + return offering; + } + } ListServiceOfferingsCmd cmd = new ListServiceOfferingsCmd(); ComponentContext.inject(cmd); cmd.setZoneId(zoneId); @@ -597,9 +638,10 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(long zoneId, int cpu return serviceOfferingDao.findByUuid(uuid); } - protected Vm createInstance(Long zoneId, Long clusterId, String name, String displayName, int cpu, long memory, - String userdata, ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { - ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zoneId, cpu, memory); + protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String accountName, Long projectId, + String name, String displayName, String serviceOfferingUuid, int cpu, long memory, String userdata, + ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { + ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(serviceOfferingUuid, zoneId, cpu, memory); if (serviceOffering == null) { throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); } @@ -608,6 +650,13 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, String dis ComponentContext.inject(cmd); cmd.setZoneId(zoneId); cmd.setClusterId(clusterId); + if (domainId != null && StringUtils.isNotEmpty(accountName)) { + cmd.setDomainId(domainId); + cmd.setAccountName(accountName); + } + if (projectId != null) { + cmd.setProjectId(projectId); + } cmd.setName(name); if (displayName != null) { cmd.setDisplayName(displayName); @@ -623,6 +672,7 @@ protected Vm createInstance(Long zoneId, Long clusterId, String name, String dis cmd.setBootMode(bootMode.toString()); } // ToDo: handle any other field? + // Handle custom offerings cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); cmd.setBlankInstance(true); Map details = new HashMap<>(); @@ -1007,6 +1057,7 @@ protected void assignVmToAccount(UserVmVO vmVO, long accountId, Pair").append(vo.getAccountUuid()).append(""); sb.append("").append(vo.getDomainUuid()).append(""); sb.append("").append(escapeText(vo.getProjectUuid())).append(""); - sb.append("").append(vo.getServiceOfferingUuid()).append(""); + if (vm.getCpuProfile() != null && StringUtils.isNotBlank(vm.getCpuProfile().getId())) { + sb.append("").append(vm.getCpuProfile().getId()).append(""); + } sb.append(""); for (DiskAttachment da : diskAttachments(vm)) { if (da == null || da.getDisk() == null || StringUtils.isBlank(da.getDisk().getId())) { continue; } - final org.apache.cloudstack.veeam.api.dto.Disk d = da.getDisk(); + final Disk d = da.getDisk(); sb.append(""); sb.append("").append(escapeText(d.getId())).append(""); sb.append("").append(d.getDiskProfile().getId()).append(""); @@ -416,62 +418,105 @@ protected static void updateFromXml(Vm vm, String ovfXml) { // Register namespace context for XPath xpath.setNamespaceContext(new OvfNamespaceContext()); + + Node contentNode = (Node) xpath.evaluate( + "//*[local-name()='Content']", + doc, + XPathConstants.NODE + ); + updateFromXmlContentNode(vm, contentNode, xpath); + Node hwSection = (Node) xpath.evaluate( "//*[local-name()='Section' and @*[local-name()='type']='ovf:VirtualHardwareSection_Type']", doc, XPathConstants.NODE ); + updateFromXmlHardwareSection(vm, hwSection, xpath); - if (hwSection != null) { - // Memory - NodeList memItems = (NodeList) xpath.evaluate( - ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='4']]", - hwSection, - XPathConstants.NODESET - ); - if (memItems != null && memItems.getLength() > 0) { - Node memItem = memItems.item(0); - String memStr = childText(memItem, "VirtualQuantity"); - if (StringUtils.isNotBlank(memStr)) { - vm.setMemory(memStr); - } - } - - // CPU - NodeList cpuItems = (NodeList) xpath.evaluate( - ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='3']]", - hwSection, - XPathConstants.NODESET - ); - if (cpuItems != null && cpuItems.getLength() > 0) { - Node cpuItem = cpuItems.item(0); - String socketsStr = childText(cpuItem, "num_of_sockets"); - String coresStr = childText(cpuItem, "cpu_per_socket"); - String threadsStr = childText(cpuItem, "threads_per_cpu"); - - if (vm.getCpu() == null) { - vm.setCpu(new Cpu()); - } - if (vm.getCpu().getTopology() == null) { - vm.getCpu().setTopology(new Topology()); - } - - if (StringUtils.isNotBlank(socketsStr)) { - vm.getCpu().getTopology().setSockets(socketsStr); - } - if (StringUtils.isNotBlank(coresStr)) { - vm.getCpu().getTopology().setCores(coresStr); - } - if (StringUtils.isNotBlank(threadsStr)) { - vm.getCpu().getTopology().setThreads(threadsStr); - } - } - } + Node metadataSection = (Node) xpath.evaluate( + "//*[local-name()='Section' and @*[local-name()='type']='ovf:CloudStackMetadata_Type']", + doc, + XPathConstants.NODE + ); + updateFromXmlCloudStackMetadataSection(vm, metadataSection, xpath); } catch (Exception e) { // Ignore parsing errors and keep original VM configuration } } + private static void updateFromXmlContentNode(Vm vm, Node contentNode, XPath xpath) { + if (contentNode == null) { + return; + } + String userId = xpathString(xpath, contentNode, "./*[local-name()='CreatedByUserId']/text()"); + if (StringUtils.isNotBlank(userId)) { + vm.setAccountId(userId); + } + String templateId = xpathString(xpath, contentNode, "./*[local-name()='TemplateId']/text()"); + if (StringUtils.isNotBlank(templateId)) { + vm.setTemplate(Ref.of("", templateId)); + } + } + + private static void updateFromXmlHardwareSection(Vm vm, Node hwSection, XPath xpath) throws XPathExpressionException { + if (hwSection == null) { + return; + } + // Memory + NodeList memItems = (NodeList) xpath.evaluate( + ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='4']]", + hwSection, + XPathConstants.NODESET + ); + if (memItems != null && memItems.getLength() > 0) { + Node memItem = memItems.item(0); + String memStr = childText(memItem, "VirtualQuantity"); + if (StringUtils.isNotBlank(memStr)) { + vm.setMemory(memStr); + } + } + + // CPU + NodeList cpuItems = (NodeList) xpath.evaluate( + ".//*[local-name()='Item'][*[local-name()='ResourceType' and text()='3']]", + hwSection, + XPathConstants.NODESET + ); + if (cpuItems != null && cpuItems.getLength() > 0) { + Node cpuItem = cpuItems.item(0); + String socketsStr = childText(cpuItem, "num_of_sockets"); + String coresStr = childText(cpuItem, "cpu_per_socket"); + String threadsStr = childText(cpuItem, "threads_per_cpu"); + + if (vm.getCpu() == null) { + vm.setCpu(new Cpu()); + } + if (vm.getCpu().getTopology() == null) { + vm.getCpu().setTopology(new Topology()); + } + + if (StringUtils.isNotBlank(socketsStr)) { + vm.getCpu().getTopology().setSockets(socketsStr); + } + if (StringUtils.isNotBlank(coresStr)) { + vm.getCpu().getTopology().setCores(coresStr); + } + if (StringUtils.isNotBlank(threadsStr)) { + vm.getCpu().getTopology().setThreads(threadsStr); + } + } + } + + private static void updateFromXmlCloudStackMetadataSection(Vm vm, Node metadataSection, XPath xpath) { + if (metadataSection == null) { + return; + } + String serviceOfferingId = xpathString(xpath, metadataSection, ".//*[local-name()='ServiceOfferingId']/text()"); + if (StringUtils.isNotBlank(serviceOfferingId)) { + vm.setCpuProfile(Ref.of("", serviceOfferingId)); + } + } + private static String xpathString(XPath xpath, Document doc, String expression) { try { String value = (String) xpath.evaluate(expression, doc, XPathConstants.STRING); @@ -481,6 +526,18 @@ private static String xpathString(XPath xpath, Document doc, String expression) } } + private static String xpathString(XPath xpath, Node node, String expression) { + if (node == null) { + return null; + } + try { + String value = (String) xpath.evaluate(expression, node, XPathConstants.STRING); + return StringUtils.isBlank(value) ? null : value.trim(); + } catch (XPathExpressionException e) { + return null; + } + } + private static String childText(Node parent, String localName) { if (parent == null || StringUtils.isBlank(localName)) { return null; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 227845a37b09..700124899dda 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -19,6 +19,7 @@ import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; @@ -71,6 +72,9 @@ public final class Vm extends BaseDto { public EmptyElement timeZone = new EmptyElement(); public EmptyElement display = new EmptyElement(); + // CloudStack-specific fields + private String accountId; + public String getName() { return name; } @@ -279,6 +283,15 @@ public void setCpuProfile(Ref cpuProfile) { this.cpuProfile = cpuProfile; } + @JsonIgnore + public String getAccountId() { + return accountId; + } + + public void setAccountId(String accountId) { + this.accountId = accountId; + } + @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Bios { diff --git a/plugins/integrations/veeam-control-service/src/main/resources/test.xml b/plugins/integrations/veeam-control-service/src/main/resources/test.xml index 8d39bd424807..5af3b9be4353 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/test.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/test.xml @@ -5,21 +5,39 @@ xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ovf:version="4.4.0.0"> - + List of networks + + +
List of Virtual Disks - + +
+
+ CloudStack specific metadata + + 644c6f0d-f6f9-11f0-9061-5254002b5a70 + 425cf134-f6f9-11f0-9061-5254002b5a70 + + 731da585-5259-46f3-bf2d-a71f62178acf + + + 5b08702c-3e4b-45fc-ba1c-425c54e69498 + 9468baee-f467-4806-9520-d313d7362694 + + +
- test-vm-abhisar - + adm-v10 + adm-v10 - 2026/01/07 13:37:09 - 2026/01/08 04:07:00 + 2026/02/26 05:36:58 + 2026/03/11 07:25:03 false guest_agent false @@ -30,12 +48,12 @@ 4.8 1 AUTO_RESUME - 1024 + 512 false false false 0 - c067a148-e4d5-11f0-98ce-00163e6c35f4 + 644c6f0d-f6f9-11f0-9061-5254002b5a70 0 false true @@ -48,32 +66,32 @@ - 4096 + 512 true false false - true + false 0 - Default - 00000000-0000-0000-0000-000000000000 - Blank + + e1a8db34-6eb4-41e0-97b8-898420437df8 + e1a8db34-6eb4-41e0-97b8-898420437df8 true 3 - 95e46398-e4d5-11f0-bb71-00163e6c35f4 + 00000000-0000-0000-0000-000000000000 2 false - 00000000-0000-0000-0000-000000000000 - Blank + e1a8db34-6eb4-41e0-97b8-898420437df8 + e1a8db34-6eb4-41e0-97b8-898420437df8 false - 2026/01/07 13:37:09 - 2026/01/07 13:38:03 + 2026/03/10 05:05:50 + 2026/02/26 05:36:58 0 -
+
Guest Operating System - other + linux
- 1 CPU, 1024 Memory + 1 CPU, 512 Memory ENGINE 4.4.0.0 @@ -85,49 +103,49 @@ 1 1 1 - 16 + 1 1 - 1024 MB of memory + 512 MB of memory Memory Size 2 4 MegaBytes - 1024 + 512 - test-vm-abhisar_Disk1 - 5cbc2ed5-de89-44a4-aa58-b7161f8afaf8 + ROOT-139 + 5b08702c-3e4b-45fc-ba1c-425c54e69498 17 - ddf18375-4c69-4ec5-8371-6dabc94e4e60/5cbc2ed5-de89-44a4-aa58-b7161f8afaf8 + 22e65515-04e6-374e-95e0-981dab9e7fe2/5b08702c-3e4b-45fc-ba1c-425c54e69498 00000000-0000-0000-0000-000000000000 - 00000000-0000-0000-0000-000000000000 + e1a8db34-6eb4-41e0-97b8-898420437df8 - 41609681-c92a-410a-bcc2-5b5e1305cdd1 - 91f4d826-e4d5-11f0-bd93-00163e6c35f4 - 2026/01/07 13:36:59 - 2026/01/07 13:53:36 - 2026/01/08 04:07:00 + 22e65515-04e6-374e-95e0-981dab9e7fe2 + 00000000-0000-0000-0000-000000000000 + 2026/02/26 05:36:58 + 2026/03/11 07:25:03 + 2026/03/11 07:25:03 disk disk {type=drive, bus=0, controller=0, target=0, unit=0} 1 true false - ua-ddf18375-4c69-4ec5-8371-6dabc94e4e60 + ua-22e65515-04e6-374e-95e0-981dab9e7fe2/5b08702c-3e4b-45fc-ba1c-425c54e69498 Ethernet adapter on [No Network] - 9a6f804d-b305-41db-b1b4-bdfd82c4b446 + 07e8e63c-13b5-4a01-9b41-6f97847d2534 10 3 - + Network-07e8e63c-13b5-4a01-9b41-6f97847d2534 true - nic1 - nic1 - 56:6f:9f:c0:00:07 + ExternalGuestNetworkGuru + ExternalGuestNetworkGuru + 02:01:00:dd:00:0c 10000 interface bridge @@ -135,7 +153,7 @@ 0 true false - ua-9a6f804d-b305-41db-b1b4-bdfd82c4b446 + ua-07e8e63c-13b5-4a01-9b41-6f97847d2534 USB Controller @@ -143,476 +161,20 @@ 23 DISABLED - - Graphical Controller - 0d4a490c-f9d7-45dd-8686-69d5bae218d6 - 20 - 1 - false - video - vga - {type=pci, slot=0x01, bus=0x00, domain=0x0000, function=0x0} - 0 - true - false - ua-0d4a490c-f9d7-45dd-8686-69d5bae218d6 - - 16384 - - - - Graphical Framebuffer - f62554f1-05fe-472e-a34b-9e6b980ad59f - 26 - graphics - vnc - - 0 - true - false - - - - CDROM - 9c38cc6a-9def-46f3-bf1c-2b3f4aa6b764 - 15 - disk - cdrom - {type=drive, bus=0, controller=0, target=0, unit=2} - 0 - true - true - ua-9c38cc6a-9def-46f3-bf1c-2b3f4aa6b764 - - - - 0 - a737450e-20b5-427e-a18b-85ec20683e31 - channel - unix - {type=virtio-serial, bus=0, controller=0, port=1} - 0 - true - false - channel0 - - - 0 - 1d3ba276-9e8d-4a16-9cdf-dfd25180b7bc - channel - unix - {type=virtio-serial, bus=0, controller=0, port=2} - 0 - true - false - channel1 - - - 0 - 8f21ce42-9499-4ded-88d4-04dff2fdc3ff - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x0, multifunction=on} - 0 - true - false - pci.1 - - 1 - pcie-root-port - - - - 0 - d1b9d421-1a57-469d-97fe-0682ad4594c3 - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x1} - 0 - true - false - pci.2 - - 2 - pcie-root-port - - - - 0 - 768c4772-eb7a-4f0f-85a7-2b94e20fe78c - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x2} - 0 - true - false - pci.3 - - 3 - pcie-root-port - - - - 0 - d20bae3b-f5d7-4131-b00a-3cf66f390434 - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x3} - 0 - true - false - pci.4 - - 4 - pcie-root-port - - - - 0 - 5887f3ad-c575-488e-9138-fca9c7064ae5 - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x4} - 0 - true - false - pci.5 - - 5 - pcie-root-port - - - - 0 - f880f086-227e-4e25-b2fc-8a3d13d1f1bd - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x5} - 0 - true - false - pci.6 - - 6 - pcie-root-port - - - - 0 - d64f62a0-6176-482b-8d24-f82fb32b8f12 - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x6} - 0 - true - false - pci.7 - - 7 - pcie-root-port - - - - 0 - 1544f32e-1e94-4e10-b198-7c5e95ab280d - controller - pci - {type=pci, slot=0x02, bus=0x00, domain=0x0000, function=0x7} - 0 - true - false - pci.8 - - 8 - pcie-root-port - - - - 0 - 7dd5080f-8c04-4593-8c6a-1dc5cd6c3e3e - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x0, multifunction=on} - 0 - true - false - pci.9 - - 9 - pcie-root-port - - - - 0 - 4dab4257-2729-482c-b4e1-6a3c05161153 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x1} - 0 - true - false - pci.10 - - 10 - pcie-root-port - - - - 0 - 99effa2f-2963-4abd-9eab-1cbe8e913ca4 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x2} - 0 - true - false - pci.11 - - 11 - pcie-root-port - - - - 0 - 2a376983-897b-4396-be32-89f2a9ca7d22 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x3} - 0 - true - false - pci.12 - - 12 - pcie-root-port - - - - 0 - 2e763d82-4475-4268-bc0a-07c915ec19c8 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x4} - 0 - true - false - pci.13 - - 13 - pcie-root-port - - - - 0 - ef39155f-760e-4374-afb9-ff05cc8b9609 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x5} - 0 - true - false - pci.14 - - 14 - pcie-root-port - - - - 0 - 74be06f0-84b6-472e-a054-486343f66084 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x6} - 0 - true - false - pci.15 - - 15 - pcie-root-port - - - - 0 - c68db43a-fa3a-4689-941d-b477d2676d27 - controller - pci - {type=pci, slot=0x03, bus=0x00, domain=0x0000, function=0x7} - 0 - true - false - pci.16 - - 16 - pcie-root-port - - - - 0 - d11cbe26-ee82-4e15-b8eb-2aa7b285d00d - controller - pci - {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x0, multifunction=on} - 0 - true - false - pci.17 - - 17 - pcie-root-port - - - - 0 - c2ef6c73-f633-41c1-8736-7e9c8d748ac2 - controller - pci - {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x1} - 0 - true - false - pci.18 - - 18 - pcie-root-port - - - - 0 - 5944d260-08c3-4f12-aa22-1e9ac76ae6c0 - controller - pci - {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x2} - 0 - true - false - pci.19 - - 19 - pcie-root-port - - - - 0 - 8c7ad6aa-ac22-4d98-86b7-45f3a13c98da - controller - pci - {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x3} - 0 - true - false - pci.20 - - 20 - pcie-root-port - - - - 0 - dc1cfae5-682d-4bb5-a53e-d604852e62cd - controller - pci - {type=pci, slot=0x04, bus=0x00, domain=0x0000, function=0x4} - 0 - true - false - pci.21 - - 21 - pcie-root-port - - - - 0 - 6117753b-8ce6-4568-8e09-c8b686396334 - controller - sata - {type=pci, slot=0x1f, bus=0x00, domain=0x0000, function=0x2} - 0 - true - false - ide - - 0 - - - - 0 - 17976687-41f8-4f7c-97f5-a76a282c40e4 - controller - virtio-serial - {type=pci, slot=0x00, bus=0x03, domain=0x0000, function=0x0} - 0 - true - false - ua-17976687-41f8-4f7c-97f5-a76a282c40e4 - - - 0 - 97f6991c-e4d5-11f0-9b4a-00163e6c35f4 + a41e097e-329a-3be5-a9e8-9bc112fe5fac rng virtio {type=pci, slot=0x00, bus=0x06, domain=0x0000, function=0x0} 0 true false - ua-97f6991c-e4d5-11f0-9b4a-00163e6c35f4 + urandom - - 0 - 0eb75625-9891-4b03-9541-c58c43c323b2 - controller - virtio-scsi - {type=pci, slot=0x00, bus=0x02, domain=0x0000, function=0x0} - 0 - true - false - ua-0eb75625-9891-4b03-9541-c58c43c323b2 - - - - - - 0 - 59536909-bac6-4202-b2ad-d84a22a41013 - balloon - memballoon - {type=pci, slot=0x00, bus=0x05, domain=0x0000, function=0x0} - 0 - true - true - ua-59536909-bac6-4202-b2ad-d84a22a41013 - - virtio - - - - 0 - e95647b0-4bb2-4ccb-b867-cbde06311038 - controller - usb - {type=pci, slot=0x00, bus=0x04, domain=0x0000, function=0x0} - 0 - true - false - ua-e95647b0-4bb2-4ccb-b867-cbde06311038 - - 0 - qemu-xhci - - -
-
- - ACTIVE - Active VM - 2026/01/07 13:37:09 -
- \ No newline at end of file + From f4a4c7a343ce973fd89197ffc61b98d245fc3848 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Mar 2026 14:32:35 +0530 Subject: [PATCH 058/129] fix for project owned resource Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 530edf4cfa90..8ce33f9481b0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -529,6 +529,9 @@ public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, bo } Ternary getVmOwner(Vm request) { + if (!VeeamControlService.InstanceRestoreAssignOwner.value()) { + return new Ternary<>(null, null, null); + } String accountUuid = request.getAccountId(); if (StringUtils.isBlank(accountUuid)) { return new Ternary<>(null, null, null); @@ -538,6 +541,7 @@ Ternary getVmOwner(Vm request) { logger.warn("Account with ID {} not found, unable to determine owner for VM creation request", accountUuid); return new Ternary<>(null, null, null); } + String accountName = account.getAccountName(); Long projectId = null; if (Account.Type.PROJECT.equals(account.getType())) { Project project = projectService.findByProjectAccountId(account.getId()); @@ -546,8 +550,9 @@ Ternary getVmOwner(Vm request) { return new Ternary<>(null, null, null); } projectId = project.getId(); + accountName = null; } - return new Ternary<>(account.getDomainId(), account.getAccountName(), projectId); + return new Ternary<>(account.getDomainId(), accountName, projectId); } public Vm createInstance(Vm request) { @@ -876,8 +881,12 @@ protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId, Pair Date: Mon, 16 Mar 2026 23:25:11 +0530 Subject: [PATCH 059/129] fix same vm restore Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 53 ++++++++++++++----- .../cloudstack/veeam/api/VmsRouteHandler.java | 30 +++++++---- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 8ce33f9481b0..2d7bddf21286 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -701,7 +701,7 @@ public Vm updateInstance(String uuid, Vm request) { return getInstance(uuid, false, false, false); } - public VmAction deleteInstance(String uuid) { + public VmAction deleteInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -718,8 +718,14 @@ public VmAction deleteInstance(String uuid) { ApiServerService.AsyncCmdResult result = apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), serviceUserAccount.second()); - AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); - return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for VM deletion"); + } + if (!async) { + waitForJobCompletion(jobVo); + } + return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete VM: " + e.getMessage(), e); } finally { @@ -727,7 +733,7 @@ public VmAction deleteInstance(String uuid) { } } - public VmAction startInstance(String uuid) { + public VmAction startInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -743,8 +749,14 @@ public VmAction startInstance(String uuid) { ApiServerService.AsyncCmdResult result = apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), serviceUserAccount.second()); - AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); - return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for VM start"); + } + if (!async) { + waitForJobCompletion(jobVo); + } + return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to start VM: " + e.getMessage(), e); } finally { @@ -752,7 +764,7 @@ public VmAction startInstance(String uuid) { } } - public VmAction stopInstance(String uuid) { + public VmAction stopInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -769,8 +781,14 @@ public VmAction stopInstance(String uuid) { ApiServerService.AsyncCmdResult result = apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), serviceUserAccount.second()); - AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); - return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for VM stop"); + } + if (!async) { + waitForJobCompletion(jobVo); + } + return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to stop VM: " + e.getMessage(), e); } finally { @@ -778,7 +796,7 @@ public VmAction stopInstance(String uuid) { } } - public VmAction shutdownInstance(String uuid) { + public VmAction shutdownInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); @@ -795,8 +813,14 @@ public VmAction shutdownInstance(String uuid) { ApiServerService.AsyncCmdResult result = apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), serviceUserAccount.second()); - AsyncJobJoinVO asyncJobJoinVO = asyncJobJoinDao.findById(result.jobId); - return AsyncJobJoinVOToJobConverter.toVmAction(asyncJobJoinVO, userVmJoinDao.findById(vo.getId())); + AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); + if (jobVo == null) { + throw new CloudRuntimeException("Failed to find job for VM shutdown"); + } + if (!async) { + waitForJobCompletion(jobVo); + } + return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to shutdown VM: " + e.getMessage(), e); } finally { @@ -1314,7 +1338,7 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { return action; } - public ResourceAction revertInstanceToSnapshot(String uuid) { + public ResourceAction revertInstanceToSnapshot(String uuid, boolean async) { ResourceAction action = null; VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); if (vo == null) { @@ -1334,6 +1358,9 @@ public ResourceAction revertInstanceToSnapshot(String uuid) { if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for snapshot revert"); } + if (!async) { + waitForJobCompletion(jobVo); + } action = AsyncJobJoinVOToJobConverter.toAction(jobVo); } catch (Exception e) { throw new CloudRuntimeException("Failed to revert to snapshot: " + e.getMessage(), e); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 22c8286878dc..e911f7636de0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -112,7 +112,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } else if ("PUT".equalsIgnoreCase(method)) { handleUpdateById(id, req, resp, outFormat, io); } else if ("DELETE".equalsIgnoreCase(method)) { - handleDeleteById(id, resp, outFormat, io); + handleDeleteById(id, req, resp, outFormat, io); } return; } else if (idAndSubPath.size() == 2) { @@ -241,6 +241,11 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.notFound(resp, null, outFormat); } + protected static boolean isRequestAsync(HttpServletRequest req) { + String asyncStr = req.getParameter("async"); + return Boolean.TRUE.toString().equals(asyncStr); + } + protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { final VmListQuery q = fromRequest(req); @@ -342,10 +347,11 @@ protected void handleUpdateById(final String id, final HttpServletRequest req, f } } - protected void handleDeleteById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleDeleteById(final String id, final HttpServletRequest req, final HttpServletResponse resp, + final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + boolean async = isRequestAsync(req); try { - VmAction vm = serverAdapter.deleteInstance(id); + VmAction vm = serverAdapter.deleteInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_OK, vm, outFormat); } catch (CloudRuntimeException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -354,8 +360,9 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + boolean async = isRequestAsync(req); try { - VmAction vm = serverAdapter.startInstance(id); + VmAction vm = serverAdapter.startInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -364,8 +371,10 @@ protected void handleStartVmById(final String id, final HttpServletRequest req, protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + boolean async = isRequestAsync(req); + String data = RouteHandler.getRequestData(req, logger); try { - VmAction vm = serverAdapter.stopInstance(id); + VmAction vm = serverAdapter.stopInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -374,8 +383,9 @@ protected void handleStopVmById(final String id, final HttpServletRequest req, f protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + boolean async = isRequestAsync(req); try { - VmAction vm = serverAdapter.shutdownInstance(id); + VmAction vm = serverAdapter.shutdownInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { io.notFound(resp, e.getMessage(), outFormat); @@ -467,8 +477,7 @@ protected void handleGetSnapshotById(final String id, final HttpServletResponse protected void handleDeleteSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String asyncStr = req.getParameter("async"); - boolean async = !Boolean.FALSE.toString().equals(asyncStr); + boolean async = isRequestAsync(req); try { ResourceAction action = serverAdapter.deleteSnapshot(id, async); if (action != null) { @@ -484,9 +493,10 @@ protected void handleDeleteSnapshotById(final String id, final HttpServletReques protected void handleRestoreSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + boolean async = isRequestAsync(req); String data = RouteHandler.getRequestData(req, logger); try { - ResourceAction response = serverAdapter.revertInstanceToSnapshot(id); + ResourceAction response = serverAdapter.revertInstanceToSnapshot(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, response, outFormat); } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); From d527762766738081c540e1db7d8f09bbf0ba8066 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 17 Mar 2026 07:35:38 +0530 Subject: [PATCH 060/129] Fix backup of stopped VMs by allowing multiple connections. --- .../resource/wrapper/LibvirtStartNBDServerCommandWrapper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index fb532fd2a9af..56d5945ced11 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -68,8 +68,10 @@ public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resour } String socketName = "/tmp/imagetransfer/" + socket + ".sock"; + // --persistent: Don't stop the service when the last client disconnects. + // --shared=NUM: Allow up to NUM clients to share the device (default 1), 0 for unlimited. Number of parallel connections is managed by the image server. String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --socket %s --persistent %s %s %s", + "systemd-run --unit=%s --property=Restart=no qemu-nbd --export-name %s --socket %s --persistent --shared=0 %s %s %s", unitName, exportName, socketName, From a6c7e55570889c091b92b65e34bf057bb31d4c58 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:23:52 +0530 Subject: [PATCH 061/129] fix export bitmap in start backup of running vm --- .../resource/wrapper/LibvirtStartBackupCommandWrapper.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 04416559c578..4ed39f1ae895 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -112,6 +112,10 @@ private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromChe if (dumpScript.execute() == null) { return null; } + if (fromCheckpointCreateTime == null) { + return new StartBackupAnswer(cmd, false, "From checkpoint create time is null for checkpoint " + fromCheckpointId); + } + String redefineXml = createCheckpointXmlForRedefine(fromCheckpointId, fromCheckpointCreateTime); File redefineFile; try { @@ -171,8 +175,7 @@ private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; xml.append(" \n"); xml.append(" \n"); From 1f72a2284c08eadc5607f8ae469b75206cdff33f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Mar 2026 17:51:20 +0530 Subject: [PATCH 062/129] changes for restore with template; refactor Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/command/user/vm/BaseDeployVMCmd.java | 243 ++---------------- .../api/command/user/vm/DeployVMCmd.java | 69 ++++- .../cloud/vm/VirtualMachineManagerImpl.java | 5 +- .../veeam/adapter/ServerAdapter.java | 31 ++- .../main/java/com/cloud/vm/UserVmManager.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 33 ++- 7 files changed, 139 insertions(+), 245 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 6ae349ca7128..aede52ed5c9a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -77,6 +77,7 @@ public class ApiConstants { public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; public static final String BIND_PASSWORD = "bindpass"; + public static final String BLANK_INSTANCE = "blankinstance"; public static final String BUS_ADDRESS = "busaddress"; public static final String BYTES_READ_RATE = "bytesreadrate"; public static final String BYTES_READ_RATE_MAX = "bytesreadratemax"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8d02dfa0a793..28e9052124ed 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -61,10 +61,10 @@ import com.cloud.network.Network.IpAddresses; import com.cloud.offering.DiskOffering; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -import com.cloud.utils.net.Dhcp; public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { @@ -75,13 +75,13 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") - private Long zoneId; + protected Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) - private String name; + protected String name; @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") - private String displayName; + protected String displayName; @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", since="4.19.0.0") @@ -89,21 +89,21 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") - private String accountName; + protected String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") - private Long domainId; + protected Long domainId; //Network information //@ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") - private List networkIds; + protected List networkIds; @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootType; + protected String bootType; @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootMode; + protected String bootMode; @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") private Boolean bootIntoSetup; @@ -138,7 +138,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") - private String hypervisor; + protected String hypervisor; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + @@ -147,7 +147,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576) - private String userData; + protected String userData; @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") private Long userdataId; @@ -189,10 +189,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String macAddress; @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,es-latam,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") - private String keyboard; + protected String keyboard; @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") - private Long projectId; + protected Long projectId; @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") private Boolean startVm; @@ -208,10 +208,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private List affinityGroupNameList; @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) - private Boolean displayVm; + protected Boolean displayVm; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") - private Map details; + protected Map details; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) private String deploymentPlanner; @@ -225,7 +225,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private Map dataDiskTemplateToDiskOfferingList; @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) - private String extraConfig; + protected String extraConfig; @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; @@ -799,217 +799,6 @@ public IoDriverPolicy getIoDriverPolicy() { return null; } - ///////////////////////////////////////////////////// - ////////////////// Setters ////////////////////////// - ///////////////////////////////////////////////////// - public void setZoneId(Long zoneId) { - this.zoneId = zoneId; - } - - public void setName(String name) { - this.name = name; - } - - public void setDisplayName(String displayName) { - this.displayName = displayName; - } - - public void setPassword(String password) { - this.password = password; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - - public void setNetworkIds(List networkIds) { - this.networkIds = networkIds; - } - - public void setBootType(String bootType) { - this.bootType = bootType; - } - - public void setBootMode(String bootMode) { - this.bootMode = bootMode; - } - - public void setBootIntoSetup(Boolean bootIntoSetup) { - this.bootIntoSetup = bootIntoSetup; - } - - public void setDiskOfferingId(Long diskOfferingId) { - this.diskOfferingId = diskOfferingId; - } - - public void setSize(Long size) { - this.size = size; - } - - public void setRootdisksize(Long rootdisksize) { - this.rootdisksize = rootdisksize; - } - - public void setDataDisksDetails(Map dataDisksDetails) { - this.dataDisksDetails = dataDisksDetails; - } - - public void setGroup(String group) { - this.group = group; - } - - public void setHypervisor(String hypervisor) { - this.hypervisor = hypervisor; - } - - public void setUserData(String userData) { - this.userData = userData; - } - - public void setUserdataId(Long userdataId) { - this.userdataId = userdataId; - } - - public void setUserdataDetails(Map userdataDetails) { - this.userdataDetails = userdataDetails; - } - - public void setSshKeyPairName(String sshKeyPairName) { - this.sshKeyPairName = sshKeyPairName; - } - - public void setSshKeyPairNames(List sshKeyPairNames) { - this.sshKeyPairNames = sshKeyPairNames; - } - - public void setHostId(Long hostId) { - this.hostId = hostId; - } - - public void setSecurityGroupIdList(List securityGroupIdList) { - this.securityGroupIdList = securityGroupIdList; - } - - public void setSecurityGroupNameList(List securityGroupNameList) { - this.securityGroupNameList = securityGroupNameList; - } - - public void setIpToNetworkList(Map ipToNetworkList) { - this.ipToNetworkList = ipToNetworkList; - } - - public void setIpAddress(String ipAddress) { - this.ipAddress = ipAddress; - } - - public void setIp6Address(String ip6Address) { - this.ip6Address = ip6Address; - } - - public void setMacAddress(String macAddress) { - this.macAddress = macAddress; - } - - public void setKeyboard(String keyboard) { - this.keyboard = keyboard; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public void setStartVm(Boolean startVm) { - this.startVm = startVm; - } - - public void setAffinityGroupIdList(List affinityGroupIdList) { - this.affinityGroupIdList = affinityGroupIdList; - } - - public void setAffinityGroupNameList(List affinityGroupNameList) { - this.affinityGroupNameList = affinityGroupNameList; - } - - public void setDisplayVm(Boolean displayVm) { - this.displayVm = displayVm; - } - - public void setDetails(Map details) { - this.details = details; - } - - public void setDeploymentPlanner(String deploymentPlanner) { - this.deploymentPlanner = deploymentPlanner; - } - - public void setDhcpOptionsNetworkList(Map dhcpOptionsNetworkList) { - this.dhcpOptionsNetworkList = dhcpOptionsNetworkList; - } - - public void setDataDiskTemplateToDiskOfferingList(Map dataDiskTemplateToDiskOfferingList) { - this.dataDiskTemplateToDiskOfferingList = dataDiskTemplateToDiskOfferingList; - } - - public void setExtraConfig(String extraConfig) { - this.extraConfig = extraConfig; - } - - public void setCopyImageTags(Boolean copyImageTags) { - this.copyImageTags = copyImageTags; - } - - public void setvAppProperties(Map vAppProperties) { - this.vAppProperties = vAppProperties; - } - - public void setvAppNetworks(Map vAppNetworks) { - this.vAppNetworks = vAppNetworks; - } - - public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { - this.dynamicScalingEnabled = dynamicScalingEnabled; - } - - public void setOverrideDiskOfferingId(Long overrideDiskOfferingId) { - this.overrideDiskOfferingId = overrideDiskOfferingId; - } - - public void setIothreadsEnabled(Boolean iothreadsEnabled) { - this.iothreadsEnabled = iothreadsEnabled; - } - - public void setIoDriverPolicy(String ioDriverPolicy) { - this.ioDriverPolicy = ioDriverPolicy; - } - - public void setNicMultiqueueNumber(Integer nicMultiqueueNumber) { - this.nicMultiqueueNumber = nicMultiqueueNumber; - } - - public void setNicPackedVirtQueues(Boolean nicPackedVirtQueues) { - this.nicPackedVirtQueues = nicPackedVirtQueues; - } - - public void setLeaseDuration(Integer leaseDuration) { - this.leaseDuration = leaseDuration; - } - - public void setLeaseExpiryAction(String leaseExpiryAction) { - this.leaseExpiryAction = leaseExpiryAction; - } - - public void setExternalDetails(Map externalDetails) { - this.externalDetails = externalDetails; - } - - public void setDataDiskInfoList(List dataDiskInfoList) { - this.dataDiskInfoList = dataDiskInfoList; - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 06b4f64b8592..f94012861929 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,10 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Stream; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -40,6 +41,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts an Instance based on a service offering, disk offering, and Template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, @@ -96,9 +98,74 @@ public boolean isBlankInstance() { return Boolean.TRUE.equals(blankInstance); } + + ///////////////////////////////////////////////////// ////////////////// Setters ////////////////////////// ///////////////////////////////////////////////////// + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setName(String name) { + this.name = name; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setNetworkIds(List networkIds) { + this.networkIds = networkIds; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public void setUserData(String userData) { + this.userData = userData; + } + + public void setKeyboard(String keyboard) { + this.keyboard = keyboard; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setDisplayVm(Boolean displayVm) { + this.displayVm = displayVm; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void setExtraConfig(String extraConfig) { + this.extraConfig = extraConfig; + } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } public void setServiceOfferingId(Long serviceOfferingId) { this.serviceOfferingId = serviceOfferingId; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 47b8eba172a0..2baf675a2579 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -50,7 +50,6 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; - import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -303,8 +302,8 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -577,7 +576,7 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - if (_userVmMgr.isBlankInstanceTemplate(template)) { + if (_userVmMgr.isBlankInstance(template)) { logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); return; } else { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 2d7bddf21286..9a7ee9ceca74 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -152,9 +152,11 @@ import com.cloud.projects.ProjectService; import com.cloud.server.ResourceTag; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.tags.ResourceTagVO; @@ -264,6 +266,9 @@ public class ServerAdapter extends ManagerBase { @Inject ServiceOfferingDao serviceOfferingDao; + @Inject + VMTemplateDao templateDao; + @Inject UserVmService userVmService; @@ -611,11 +616,15 @@ public Vm createInstance(Vm request) { if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { serviceOfferingUuid = request.getCpuProfile().getId(); } + String templateUuid = null; + if (request.getTemplate() != null && StringUtils.isNotEmpty(request.getTemplate().getId())) { + templateUuid = request.getTemplate().getId(); + } Pair serviceUserAccount = getServiceAccount(); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createInstance(zoneId, clusterId, owner.first(), owner.second(), owner.third(), name, displayName, - serviceOfferingUuid, cpu, memory, userdata, bootType, bootMode); + serviceOfferingUuid, cpu, memory, templateUuid, userdata, bootType, bootMode); } finally { CallContext.unregister(); } @@ -643,9 +652,21 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(String serviceOfferi return serviceOfferingDao.findByUuid(uuid); } + protected VMTemplateVO getTemplateForVmCreation(String templateUuid) { + if (StringUtils.isBlank(templateUuid)) { + return null; + } + VMTemplateVO template = templateDao.findByUuid(templateUuid); + if (template == null) { + logger.warn("Template with ID {} not found, VM will be created with default template", templateUuid); + return null; + } + return template; + } + protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String accountName, Long projectId, - String name, String displayName, String serviceOfferingUuid, int cpu, long memory, String userdata, - ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { + String name, String displayName, String serviceOfferingUuid, int cpu, long memory, String templateUuid, + String userdata, ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(serviceOfferingUuid, zoneId, cpu, memory); if (serviceOffering == null) { throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); @@ -676,6 +697,10 @@ protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String a if (bootMode != null) { cmd.setBootMode(bootMode.toString()); } + VMTemplateVO template = getTemplateForVmCreation(templateUuid); + if (template != null) { + cmd.setTemplateId(template.getId()); + } // ToDo: handle any other field? // Handle custom offerings cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index fed8de36c3de..69f11b41d1fa 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -204,5 +204,5 @@ static Set getStrictHostTags() { */ boolean isVMPartOfAnyCKSCluster(VMInstanceVO vm); - boolean isBlankInstanceTemplate(VirtualMachineTemplate template); + boolean isBlankInstance(VirtualMachineTemplate template); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 5a5f127d050e..844e44ca44bd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -3933,8 +3933,8 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service _accountMgr.checkAccess(owner, _diskOfferingDao.findById(diskOfferingId), zone); // If no network is specified, find system security group enabled network - if (isBlankInstanceTemplate(template)) { - logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced security group enabled zone", hypervisor); + if (isBlankInstance(template)) { + logger.debug("Blank instance for {} hypervisor, skipping network allocation in an advanced security group enabled zone", hypervisor); } else if (networkIdList == null || networkIdList.isEmpty()) { Network networkWithSecurityGroup = _networkModel.getNetworkWithSGWithFreeIPs(owner, zone.getId()); if (networkWithSecurityGroup == null) { @@ -4048,7 +4048,7 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv _accountMgr.checkAccess(owner, diskOffering, zone); List vpcSupportedHTypes = _vpcMgr.getSupportedVpcHypervisors(); - if (isBlankInstanceTemplate(template)) { + if (isBlankInstance(template)) { logger.debug("Template is a dummy template for hypervisor {}, skipping network allocation in an advanced zone", hypervisor); } else if (networkIdList == null || networkIdList.isEmpty()) { NetworkVO defaultNetwork = getDefaultNetwork(zone, owner, false); @@ -4485,7 +4485,8 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri } } - if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType) && !isBlankInstanceTemplate(template)) { + if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && + !SHAREDFSVM.equals(vmType) && !isBlankInstanceDefaultTemplate(template)) { throw new InvalidParameterValueException(String.format("Unable to use system template %s to deploy a user vm", template)); } @@ -4498,7 +4499,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri if (CollectionUtils.isEmpty(snapshotsOnZone)) { throw new InvalidParameterValueException("The snapshot does not exist on zone " + zone.getId()); } - } else if (!isBlankInstanceTemplate(template)) { + } else if (!isBlankInstanceDefaultTemplate(template)) { List listZoneTemplate = _templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { throw new InvalidParameterValueException("The template " + template.getId() + " is not available for use"); @@ -4613,7 +4614,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri // by Agent Manager in order to configure default // gateway for the vm if (defaultNetworkNumber == 0) { - if (isBlankInstanceTemplate(template)) { + if (isBlankInstance(template)) { logger.debug("Template is a dummy template for hypervisor {}, vm can be created without a default network", hypervisorType); } else { throw new InvalidParameterValueException("At least 1 default network has to be specified for the vm"); @@ -6483,7 +6484,11 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE (!(HypervisorType.KVM.equals(template.getHypervisorType()) || HypervisorType.KVM.equals(cmd.getHypervisor())))) { throw new InvalidParameterValueException("Deploying a virtual machine with existing volume/snapshot is supported only from KVM hypervisors"); } - if (template == null && HypervisorType.KVM.equals(cmd.getHypervisor()) && cmd.isBlankInstance()) { + boolean blankInstance = cmd.isBlankInstance(); + if (blankInstance) { + CallContext.current().putContextParameter(ApiConstants.BLANK_INSTANCE, true); + } + if (template == null && HypervisorType.KVM.equals(cmd.getHypervisor()) && blankInstance) { template = getBlankInstanceTemplate(); logger.info("Creating launch permission for Dummy template"); LaunchPermissionVO launchPermission = new LaunchPermissionVO(template.getId(), owner.getId()); @@ -6648,7 +6653,7 @@ private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Accoun applyLeaseOnCreateInstance(vm, cmd.getLeaseDuration(), cmd.getLeaseExpiryAction(), svcOffering); } - if (isBlankInstanceTemplate(template) && cmd instanceof DeployVMCmd && ((DeployVMCmd) cmd).isBlankInstance()) { + if (isBlankInstance(template) && cmd instanceof DeployVMCmd && ((DeployVMCmd) cmd).isBlankInstance()) { logger.info("Revoking launch permission for Dummy template"); launchPermissionDao.removePermissions(template.getId(), Collections.singletonList(owner.getId())); } @@ -10091,11 +10096,19 @@ private void setVncPasswordForKvmIfAvailable(Map customParameter } } - @Override - public boolean isBlankInstanceTemplate(VirtualMachineTemplate template) { + protected boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template) { return KVM_VM_DUMMY_TEMPLATE_NAME.equals(template.getUniqueName()); } + @Override + public boolean isBlankInstance(VirtualMachineTemplate template) { + if (isBlankInstanceDefaultTemplate(template)) { + return true; + } + return MapUtils.getBoolean(CallContext.current().getContextParameters(), + ApiConstants.BLANK_INSTANCE); + } + VMTemplateVO getBlankInstanceTemplate() { VMTemplateVO template = _templateDao.findByName(KVM_VM_DUMMY_TEMPLATE_NAME); if (template != null) { From 3bce25db2b7353385c8e740e67a3dc48e80c56cd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 18 Mar 2026 12:05:23 +0530 Subject: [PATCH 063/129] fix check for blank instance Signed-off-by: Abhishek Kumar --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 844e44ca44bd..58e5fd799770 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -10105,8 +10105,8 @@ public boolean isBlankInstance(VirtualMachineTemplate template) { if (isBlankInstanceDefaultTemplate(template)) { return true; } - return MapUtils.getBoolean(CallContext.current().getContextParameters(), - ApiConstants.BLANK_INSTANCE); + return Boolean.TRUE.equals( + MapUtils.getBoolean(CallContext.current().getContextParameters(), ApiConstants.BLANK_INSTANCE)); } VMTemplateVO getBlankInstanceTemplate() { From 90d87d0e9234528c70b9c9b305b5fb822f073a32 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 18 Mar 2026 14:24:57 +0530 Subject: [PATCH 064/129] restore with correct bios type Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 24 ++++--- .../AsyncJobJoinVOToJobConverter.java | 2 +- .../converter/UserVmJoinVOToVmConverter.java | 19 +++-- .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 16 ++--- .../apache/cloudstack/veeam/api/dto/Vm.java | 70 +++++++++++++++++++ 5 files changed, 101 insertions(+), 30 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 9a7ee9ceca74..6e252829bad6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -178,6 +178,7 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -239,6 +240,9 @@ public class ServerAdapter extends ManagerBase { @Inject UserVmJoinDao userVmJoinDao; + @Inject + VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject VolumeDao volumeDao; @@ -519,7 +523,7 @@ public VnicProfile getVnicProfile(String uuid) { public List listAllInstances() { List vms = userVmJoinDao.listAll(); - return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById); + return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); } public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, boolean allContent) { @@ -528,6 +532,7 @@ public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, bo throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, + this::getDetailsByInstanceId, includeDisks ? this::listDiskAttachmentsByInstanceId : null, includeNics ? this::listNicsByInstance : null, allContent); @@ -605,12 +610,7 @@ public Vm createInstance(Vm request) { if (request.getInitialization() != null) { userdata = request.getInitialization().getCustomScript(); } - ApiConstants.BootType bootType = ApiConstants.BootType.BIOS; - ApiConstants.BootMode bootMode = ApiConstants.BootMode.LEGACY; - if (request.getBios() != null && StringUtils.isNotEmpty(request.getBios().getType()) && request.getBios().getType().contains("secure")) { - bootType = ApiConstants.BootType.UEFI; - bootMode = ApiConstants.BootMode.SECURE; - } + Pair bootOptions = Vm.Bios.retrieveBootOptions(request.getBios()); Ternary owner = getVmOwner(request); String serviceOfferingUuid = null; if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { @@ -624,7 +624,7 @@ public Vm createInstance(Vm request) { CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { return createInstance(zoneId, clusterId, owner.first(), owner.second(), owner.third(), name, displayName, - serviceOfferingUuid, cpu, memory, templateUuid, userdata, bootType, bootMode); + serviceOfferingUuid, cpu, memory, templateUuid, userdata, bootOptions.first(), bootOptions.second()); } finally { CallContext.unregister(); } @@ -714,8 +714,8 @@ protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String a UserVm vm = userVmService.createVirtualMachine(cmd); vm = userVmService.finalizeCreateVirtualMachine(vm.getId()); UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); - return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::listDiskAttachmentsByInstanceId, - this::listNicsByInstance, false); + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, + this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); } @@ -1266,6 +1266,10 @@ protected NetworkVO getNetworkById(Long networkId) { return networkDao.findById(networkId); } + protected Map getDetailsByInstanceId(Long instanceId) { + return vmInstanceDetailsDao.listDetailsKeyPairs(instanceId, true); + } + public List listAllJobs() { Pair serviceUserAccount = getServiceAccount(); List jobIds = asyncJobDao.listPendingJobIdsForAccount(serviceUserAccount.second().getId()); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index 625c9d9e469a..dc2853dfd766 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -98,7 +98,7 @@ protected static void fillAction(final ResourceAction action, final AsyncJobJoin public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { VmAction action = new VmAction(); fillAction(action, vo); - action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null, false)); + action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null, null, false)); return action; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 42431dc357b3..44691a0ef492 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -20,9 +20,11 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.api.ApiService; import org.apache.cloudstack.veeam.api.VmsRouteHandler; @@ -37,6 +39,7 @@ import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.api.query.vo.HostJoinVO; @@ -54,6 +57,7 @@ private UserVmJoinVOToVmConverter() { * @param src UserVmJoinVO */ public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, + final Function> detailsResolver, final Function> disksResolver, final Function> nicsResolver, final boolean allContent) { @@ -124,11 +128,11 @@ public static Vm toVm(final UserVmJoinVO src, final Function h boot.setDevices(NamedList.of("device", List.of("hd"))); os.setBoot(boot); dst.setOs(os); - Vm.Bios bios = new Vm.Bios(); - bios.setType("q35_secure_boot"); - Vm.Bios.BootMenu bootMenu = new Vm.Bios.BootMenu(); - bootMenu.setEnabled("false"); - bios.setBootMenu(bootMenu); + Vm.Bios bios = Vm.Bios.getDefault(); + if (detailsResolver != null) { + Map details = detailsResolver.apply(src.getId()); + Vm.Bios.updateBios(bios, MapUtils.getString(details, ApiConstants.BootType.UEFI.toString())); + } dst.setBios(bios); dst.setType("desktop"); dst.setOrigin("ovirt"); @@ -176,9 +180,10 @@ private static Vm.Initialization getOvfInitialization(Vm vm, UserVmJoinVO vo) { return initialization; } - public static List toVmList(final List srcList, final Function hostResolver) { + public static List toVmList(final List srcList, final Function hostResolver, + final Function> detailsResolver) { return srcList.stream() - .map(v -> toVm(v, hostResolver, null, null, false)) + .map(v -> toVm(v, hostResolver, detailsResolver, null, null, false)) .collect(Collectors.toList()); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java index 8ca75a27485c..fcccf299f27f 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -230,7 +230,7 @@ public static String toXml(final Vm vm, final UserVmJoinVO vo) { sb.append("LOCK_SCREEN"); sb.append("0"); sb.append(""); - sb.append("").append(mapBiosType(vm.getBios() != null ? vm.getBios().getType() : null)).append(""); + sb.append("").append(vm.getBios() != null ? vm.getBios().getTypeOrdinal() : 1).append(""); sb.append(""); sb.append(""); sb.append(""); @@ -456,6 +456,9 @@ private static void updateFromXmlContentNode(Vm vm, Node contentNode, XPath xpat if (StringUtils.isNotBlank(templateId)) { vm.setTemplate(Ref.of("", templateId)); } + String biosType = xpathString(xpath, contentNode, "./*[local-name()='BiosType']/text()"); + Vm.Bios bios = Vm.Bios.getBiosFromOrdinal(biosType); + vm.setBios(bios); } private static void updateFromXmlHardwareSection(Vm vm, Node hwSection, XPath xpath) throws XPathExpressionException { @@ -646,17 +649,6 @@ private static String mapVolumeType(String sparse) { return "true".equalsIgnoreCase(sparse) ? "Sparse" : "Preallocated"; } - private static int mapBiosType(String biosType) { - if (StringUtils.isBlank(biosType)) { - return 2; - } - String t = biosType.toLowerCase(Locale.ROOT); - if (t.contains("uefi") || t.contains("secure")) { - return 2; - } - return 0; - } - private static String mapBalloonEnabled(Vm vm) { if (vm.getMemoryPolicy() == null || vm.getMemoryPolicy().getBallooning() == null) { return "true"; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 700124899dda..c6ade15853e3 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -19,6 +19,10 @@ import java.util.List; +import org.apache.cloudstack.api.ApiConstants; + +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -302,6 +306,18 @@ public String getType() { return type; } + @JsonIgnore + public int getTypeOrdinal() { + switch (type) { + case "q35_secure_boot": + return 4; + case "q35_ovmf": + return 2; + default: + return 1; // default to i440fx_sea_bios + } + } + public void setType(String type) { this.type = type; } @@ -327,6 +343,60 @@ public void setEnabled(String enabled) { this.enabled = enabled; } } + + public static Bios getDefault() { + Bios bios = new Bios(); + bios.setType("i440fx_sea_bios"); + BootMenu bootMenu = new BootMenu(); + bootMenu.setEnabled("false"); + bios.setBootMenu(bootMenu); + return bios; + } + + public static void updateBios(Bios bios, String bootMode) { + if (StringUtils.isEmpty(bootMode)) { + return; + } + if (ApiConstants.BootMode.SECURE.toString().equals(bootMode)) { + bios.setType("q35_secure_boot"); + return; + } + bios.setType("q35_ovmf"); + } + + public static Bios getBiosFromOrdinal(String bootTypeStr) { + Bios bios = getDefault(); + if (StringUtils.isEmpty(bootTypeStr)) { + return bios; + } + int type = 1; + try { + type = Integer.parseInt(bootTypeStr); + } catch (NumberFormatException e) { + return bios; + } + if (type == 2 || type == 3) { + bios.setType("q35_ovmf"); + } else if (type == 4) { + bios.setType("q35_secure_boot"); + } + return bios; + } + + public static Pair retrieveBootOptions(Bios bios) { + Pair defaultValue = + new Pair<>(ApiConstants.BootType.BIOS, ApiConstants.BootMode.LEGACY); + if (bios == null || StringUtils.isEmpty(bios.getType())) { + return defaultValue; + } + if ("q35_secure_boot".equals(bios.getType())) { + return new Pair<>(ApiConstants.BootType.UEFI, ApiConstants.BootMode.SECURE); + } + if (bios.getType().startsWith("q35_")) { + return new Pair<>(ApiConstants.BootType.UEFI, ApiConstants.BootMode.LEGACY); + } + return defaultValue; + } } @JsonInclude(JsonInclude.Include.NON_NULL) From 1e9a116bcb9331b9623cf2b7b39a5e88f46539e8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 18 Mar 2026 18:50:29 +0530 Subject: [PATCH 065/129] fix naming issue Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/veeam/adapter/ServerAdapter.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 6e252829bad6..e463c02cdb01 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -575,9 +575,7 @@ public Vm createInstance(Vm request) { throw new InvalidParameterValueException("Invalid name specified for the VM"); } String displayName = name; - if (name.endsWith("_restored")) { - name = name.replace("_restored", "-restored"); - } + name = name.replaceAll("_", "-"); Long zoneId = null; Long clusterId = null; if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { From cb2d7360327f804330705e19fb63dacd9bf9c56d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 18 Mar 2026 18:51:08 +0530 Subject: [PATCH 066/129] changes for default bios boot type Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/veeam/api/dto/Vm.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index c6ade15853e3..ccf496db192c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.api.ApiConstants; +import com.cloud.utils.EnumUtils; import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -299,6 +300,14 @@ public void setAccountId(String accountId) { @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Bios { + public enum Type { + cluster_default, + i440fx_sea_bios, + q35_ovmf, + q35_sea_bios, + q35_secure_boot + } + private String type; // "uefi" or "bios" or whatever mapping you choose private BootMenu bootMenu = new BootMenu(); @@ -308,14 +317,8 @@ public String getType() { @JsonIgnore public int getTypeOrdinal() { - switch (type) { - case "q35_secure_boot": - return 4; - case "q35_ovmf": - return 2; - default: - return 1; // default to i440fx_sea_bios - } + Type enumType = EnumUtils.fromString(Type.class, type, Type.q35_sea_bios); + return enumType.ordinal(); } public void setType(String type) { @@ -346,7 +349,7 @@ public void setEnabled(String enabled) { public static Bios getDefault() { Bios bios = new Bios(); - bios.setType("i440fx_sea_bios"); + bios.setType(Type.q35_sea_bios.name()); BootMenu bootMenu = new BootMenu(); bootMenu.setEnabled("false"); bios.setBootMenu(bootMenu); @@ -358,10 +361,10 @@ public static void updateBios(Bios bios, String bootMode) { return; } if (ApiConstants.BootMode.SECURE.toString().equals(bootMode)) { - bios.setType("q35_secure_boot"); + bios.setType(Type.q35_secure_boot.name()); return; } - bios.setType("q35_ovmf"); + bios.setType(Type.q35_ovmf.name()); } public static Bios getBiosFromOrdinal(String bootTypeStr) { @@ -375,10 +378,11 @@ public static Bios getBiosFromOrdinal(String bootTypeStr) { } catch (NumberFormatException e) { return bios; } - if (type == 2 || type == 3) { - bios.setType("q35_ovmf"); - } else if (type == 4) { - bios.setType("q35_secure_boot"); + + if (type == Type.q35_ovmf.ordinal()) { + bios.setType(Type.q35_ovmf.name()); + } else if (type == Type.q35_secure_boot.ordinal()) { + bios.setType(Type.q35_secure_boot.name()); } return bios; } @@ -389,10 +393,10 @@ public static Pair retrieveBootOpt if (bios == null || StringUtils.isEmpty(bios.getType())) { return defaultValue; } - if ("q35_secure_boot".equals(bios.getType())) { + if (Type.q35_secure_boot.name().equals(bios.getType())) { return new Pair<>(ApiConstants.BootType.UEFI, ApiConstants.BootMode.SECURE); } - if (bios.getType().startsWith("q35_")) { + if (Type.q35_ovmf.name().equals(bios.getType())) { return new Pair<>(ApiConstants.BootType.UEFI, ApiConstants.BootMode.LEGACY); } return defaultValue; From 38c8b70cf3500940044a5e8e1b16bd1cbcee6223 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 20 Mar 2026 14:03:17 +0530 Subject: [PATCH 067/129] server,engine-schema: allow retrieving volume stats for stopped vms Earlier, we were finding only those instance which have host_id equal to the given host. Changed code now also returns those VMs which have host_id as NULL and last_host_id as the given host. Signed-off-by: Abhishek Kumar --- .../java/com/cloud/vm/dao/VMInstanceDao.java | 2 ++ .../java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 16 ++++++++++++++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 23541c2431e7..06ae01e92fa0 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -192,4 +192,6 @@ List searchRemovedByRemoveDate(final Date startDate, final Date en int getVmCountByOfferingNotInDomain(Long serviceOfferingId, List domainIds); List listByIdsIncludingRemoved(List ids); + + List listIdsByHostIdForVolumeStats(long hostIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 589a63ea0d84..96b073522247 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -1296,4 +1296,20 @@ public List listByIdsIncludingRemoved(List ids) { sc.setParameters("ids", ids.toArray()); return listIncludingRemovedBy(sc); } + + @Override + public List listIdsByHostIdForVolumeStats(long hostId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.selectFields(sb.entity().getId()); + sb.and().op("host", sb.entity().getHostId(), SearchCriteria.Op.EQ); + sb.or().op("hostNull", sb.entity().getHostId(), Op.NULL); + sb.and("lastHost", sb.entity().getLastHostId(), SearchCriteria.Op.EQ); + sb.cp(); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("host", hostId); + sc.setParameters("lastHost", hostId); + return customSearch(sc, null); + } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 58e5fd799770..2ab4156a1aea 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2343,9 +2343,9 @@ public HashMap getVolumeStatistics(long clusterId, Str } private List getVolumesByHost(HostVO host, StoragePool pool) { - List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); + List vmsPerHost = _vmInstanceDao.listIdsByHostIdForVolumeStats(host.getId()); return vmsPerHost.stream() - .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> + .flatMap(vmId -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vmId,pool.getId()).stream().map(vol -> vol.getState() == Volume.State.Ready ? (vol.getFormat() == ImageFormat.OVA ? vol.getChainInfo() : vol.getPath()) : null).filter(Objects::nonNull)) .collect(Collectors.toList()); } From 50403f75486ccd6e1b7eec62a1722b0384ba305e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 20 Mar 2026 15:14:36 +0530 Subject: [PATCH 068/129] changes for allowed cidrs; refactor Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/VeeamControlServer.java | 15 ++- .../cloudstack/veeam/VeeamControlService.java | 15 ++- .../veeam/VeeamControlServiceImpl.java | 59 ++++++++-- .../filter/AllowedClientCidrsFilter.java | 100 +++++++++++++++++ .../veeam/filter/BearerOrBasicAuthFilter.java | 83 ++++++-------- .../cloudstack/veeam/sso/SsoService.java | 103 +++++------------- .../cloudstack/veeam/utils/DataUtil.java | 44 ++++++++ .../cloudstack/veeam/utils/JwtUtil.java | 57 ++++++++++ 8 files changed, 333 insertions(+), 143 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/DataUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java index adf9e45ecdf7..3121fd6ecf4c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.filter.AllowedClientCidrsFilter; import org.apache.cloudstack.veeam.filter.BearerOrBasicAuthFilter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -43,6 +44,7 @@ import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -51,12 +53,14 @@ public class VeeamControlServer { private static final Logger LOGGER = LogManager.getLogger(VeeamControlServer.class); + private final VeeamControlService veeamControlService; private Server server; private List routeHandlers; - public VeeamControlServer(List routeHandlers) { + public VeeamControlServer(List routeHandlers, VeeamControlService veeamControlService) { this.routeHandlers = new ArrayList<>(routeHandlers); this.routeHandlers.sort((a, b) -> Integer.compare(b.priority(), a.priority())); + this.veeamControlService = veeamControlService; } public void startIfEnabled() throws Exception { @@ -118,8 +122,15 @@ public void startIfEnabled() throws Exception { new ServletContextHandler(ServletContextHandler.NO_SESSIONS); ctx.setContextPath(ctxPath); + // CIDR filter for all routes + AllowedClientCidrsFilter cidrFilter = new AllowedClientCidrsFilter(veeamControlService); + FilterHolder cidrHolder = new FilterHolder(cidrFilter); + ctx.addFilter(cidrHolder, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); + // Bearer or Basic Auth for all routes - ctx.addFilter(BearerOrBasicAuthFilter.class, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); + BearerOrBasicAuthFilter authFilter = new BearerOrBasicAuthFilter(veeamControlService); + FilterHolder authHolder = new FilterHolder(authFilter); + ctx.addFilter(authHolder, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); // Front controller servlet ctx.addServlet(new ServletHolder(new VeeamControlServlet(routeHandlers)), "/*"); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java index 38e350d59998..8e4abef9743f 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.veeam; +import java.util.List; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -31,9 +33,9 @@ public interface VeeamControlService extends PluggableService, Configurable { "8090", "Port for Veeam Integration REST API server", false); ConfigKey ContextPath = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.context.path", "/ovirt-engine", "Context path for Veeam Integration REST API server", false); - ConfigKey Username = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.api.username", + ConfigKey Username = new ConfigKey<>("Secure", String.class, "integration.veeam.control.api.username", "veeam", "Username for Basic Auth on Veeam Integration REST API server", true); - ConfigKey Password = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.api.password", + ConfigKey Password = new ConfigKey<>("Secure", String.class, "integration.veeam.control.api.password", "change-me", "Password for Basic Auth on Veeam Integration REST API server", true); ConfigKey ServiceAccountId = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.service.account", "", @@ -46,4 +48,13 @@ public interface VeeamControlService extends PluggableService, Configurable { "false", "Attempt to assign restored Instance to the owner based on OVF and network " + "details. If the assignment fails or set to false then the Instance will remain owned by the service " + "account", true); + ConfigKey AllowedClientCidrs = new ConfigKey<>("Advanced", String.class, + "integration.veeam.control.allowed.client.cidrs", + "", "Comma-separated list of CIDR blocks representing clients allowed to access the API. " + + "If empty, all clients will be allowed. Example: '192.168.1.1/24,192.168.2.100/32", true); + + + List getAllowedClientCidrs(); + + boolean validateCredentials(String username, String password); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java index 683d0052f9d3..a00d6bd5b836 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java @@ -17,17 +17,43 @@ package org.apache.cloudstack.veeam; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.cache.SingleCache; +import org.apache.cloudstack.veeam.utils.DataUtil; +import org.apache.commons.lang3.StringUtils; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.net.NetUtils; public class VeeamControlServiceImpl extends ManagerBase implements VeeamControlService { private List routeHandlers; - private VeeamControlServer veeamControlServer; + private SingleCache> allowedClientCidrsCache; + + protected List getAllowedClientCidrsInternal() { + String allowedClientCidrsStr = AllowedClientCidrs.value(); + if (StringUtils.isBlank(allowedClientCidrsStr)) { + return Collections.emptyList(); + } + List allowedClientCidrs = List.of(allowedClientCidrsStr.split(",")); + // Sanitize and remove any incorrect CIDR entries + allowedClientCidrs = allowedClientCidrs.stream() + .map(String::trim) + .filter(StringUtils::isNotBlank) + .filter(cidr -> { + boolean valid = NetUtils.isValidIp4Cidr(cidr); + if (!valid) { + logger.warn("Invalid CIDR entry '{}' in allowed client CIDRs, ignoring", cidr); + } + return valid; + }).collect(Collectors.toList()); + return allowedClientCidrs; + } public List getRouteHandlers() { return routeHandlers; @@ -37,9 +63,21 @@ public void setRouteHandlers(final List routeHandlers) { this.routeHandlers = routeHandlers; } + @Override + public List getAllowedClientCidrs() { + return allowedClientCidrsCache.get(); + } + + @Override + public boolean validateCredentials(String username, String password) { + return DataUtil.constantTimeEquals(Username.value(), username) && + DataUtil.constantTimeEquals(Password.value(), password); + } + @Override public boolean start() { - veeamControlServer = new VeeamControlServer(getRouteHandlers()); + allowedClientCidrsCache = new SingleCache<>(30, this::getAllowedClientCidrsInternal); + veeamControlServer = new VeeamControlServer(getRouteHandlers(), this); try { veeamControlServer.startIfEnabled(); } catch (Exception e) { @@ -71,14 +109,15 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { - Enabled, - BindAddress, - Port, - ContextPath, - Username, - Password, - ServiceAccountId, - InstanceRestoreAssignOwner + Enabled, + BindAddress, + Port, + ContextPath, + Username, + Password, + ServiceAccountId, + InstanceRestoreAssignOwner, + AllowedClientCidrs }; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilter.java new file mode 100644 index 000000000000..9c3c199704e7 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilter.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.filter; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.net.NetUtils; + +public class AllowedClientCidrsFilter implements Filter { + + private static final Logger LOGGER = LogManager.getLogger(AllowedClientCidrsFilter.class); + + private final VeeamControlService veeamControlService; + + public AllowedClientCidrsFilter(VeeamControlService veeamControlService) { + this.veeamControlService = veeamControlService; + } + + @Override + public void init(FilterConfig filterConfig) { + // no-op + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + if (!(request instanceof HttpServletRequest) || !(response instanceof HttpServletResponse)) { + chain.doFilter(request, response); + return; + } + + final HttpServletRequest req = (HttpServletRequest) request; + final HttpServletResponse resp = (HttpServletResponse) response; + + if (veeamControlService == null) { + LOGGER.warn("Failed to inject VeeamControlService, allowing request by default"); + chain.doFilter(request, response); + return; + } + + final List cidrList = veeamControlService.getAllowedClientCidrs(); + if (CollectionUtils.isEmpty(cidrList)) { + chain.doFilter(request, response); + return; + } + + final String remoteAddr = req.getRemoteAddr(); + try { + final InetAddress clientIp = InetAddress.getByName(remoteAddr); + final boolean allowed = NetUtils.isIpInCidrList(clientIp, cidrList.toArray(new String[0])); + if (!allowed) { + LOGGER.warn("Rejected request from client IP {} not in allowed CIDRs {}", remoteAddr, cidrList); + resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden"); + return; + } + } catch (Exception e) { + LOGGER.warn("Rejected request failed to parse client IP {}: {}", remoteAddr, e.getMessage()); + resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden"); + return; + } + + chain.doFilter(request, response); + } + + @Override + public void destroy() { + // no-op + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java index 511e89ec68c7..e86bd6a2a3ef 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java @@ -21,11 +21,8 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Base64; -import java.util.List; import java.util.Map; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -36,22 +33,30 @@ import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.sso.SsoService; +import org.apache.cloudstack.veeam.utils.DataUtil; +import org.apache.cloudstack.veeam.utils.JwtUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; public class BearerOrBasicAuthFilter implements Filter { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - // Keep these aligned with SsoService (move to ConfigKeys later) - public static final List REQUIRED_SCOPES = List.of("ovirt-app-admin", "ovirt-app-portal"); - public static final String ISSUER = "veeam-control"; - public static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; + private final VeeamControlService veeamControlService; - private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + public BearerOrBasicAuthFilter(VeeamControlService veeamControlService) { + this.veeamControlService = veeamControlService; + } + + @Override + public void init(FilterConfig filterConfig) { + } - @Override public void init(FilterConfig filterConfig) {} - @Override public void destroy() {} + @Override + public void destroy() { + } @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) @@ -89,9 +94,6 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } private boolean verifyBasic(String b64) { - final String expectedUser = VeeamControlService.Username.value(); - final String expectedPass = VeeamControlService.Password.value(); - final String decoded; try { decoded = new String(Base64.getDecoder().decode(b64), StandardCharsets.UTF_8); @@ -105,7 +107,7 @@ private boolean verifyBasic(String b64) { final String user = decoded.substring(0, idx); final String pass = decoded.substring(idx + 1); - return constantTimeEquals(user, expectedUser) && constantTimeEquals(pass, expectedPass); + return veeamControlService != null && veeamControlService.validateCredentials(user, pass); } /** @@ -114,9 +116,6 @@ private boolean verifyBasic(String b64) { * - "iss" matches * - "exp" not expired * - "scope" contains REQUIRED_SCOPES (space-separated) - * - * NOTE: This does not parse JSON robustly; it’s sufficient for the token you mint in SsoService. - * If you want robust parsing, switch to Nimbus and keep the rest the same. */ private boolean verifyJwtHs256(String token) { final String[] parts = token.split("\\."); @@ -128,8 +127,8 @@ private boolean verifyJwtHs256(String token) { final byte[] expectedSig; try { - expectedSig = hmacSha256((headerB64 + "." + payloadB64).getBytes(StandardCharsets.UTF_8), - HMAC_SECRET.getBytes(StandardCharsets.UTF_8)); + expectedSig = JwtUtil.hmacSha256((headerB64 + "." + payloadB64).getBytes(StandardCharsets.UTF_8), + SsoService.HMAC_SECRET.getBytes(StandardCharsets.UTF_8)); } catch (Exception e) { return false; } @@ -141,21 +140,22 @@ private boolean verifyJwtHs256(String token) { return false; } - if (!constantTimeEquals(expectedSig, providedSig)) return false; + if (!DataUtil.constantTimeEquals(expectedSig, providedSig)) return false; Map payloadMap; try { String payloadJson = new String(Base64.getUrlDecoder().decode(payloadB64), StandardCharsets.UTF_8); payloadMap = JSON_MAPPER.readValue( payloadJson, - new TypeReference<>() {} + new TypeReference<>() { + } ); } catch (IllegalArgumentException | JsonProcessingException e) { return false; } - final String iss = (String)payloadMap.get("iss"); - final String scope = (String)payloadMap.get("scope"); + final String iss = (String) payloadMap.get("iss"); + final String scope = (String) payloadMap.get("scope"); final Object expObj = payloadMap.get("exp"); Long exp = null; if (expObj instanceof Number) { @@ -163,10 +163,11 @@ private boolean verifyJwtHs256(String token) { } else if (expObj instanceof String) { try { exp = Long.parseLong((String) expObj); - } catch (NumberFormatException ignored) {} + } catch (NumberFormatException ignored) { + } } - if (!ISSUER.equals(iss)) { + if (!JwtUtil.ISSUER.equals(iss)) { return false; } if (exp == null || Instant.now().getEpochSecond() >= exp) { @@ -177,7 +178,7 @@ private boolean verifyJwtHs256(String token) { private static boolean hasRequiredScopes(String scope) { String[] scopes = scope.split("\\s+"); - for (String required : REQUIRED_SCOPES) { + for (String required : SsoService.REQUIRED_SCOPES) { if (!hasScope(scopes, required)) return false; } return true; @@ -192,22 +193,15 @@ private static boolean hasScope(String[] scopes, String required) { return false; } - private static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { - final Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(key, "HmacSHA256")); - return mac.doFinal(data); - } - private static void unauthorized(HttpServletRequest req, HttpServletResponse resp, String error, String desc) throws IOException { - - // IMPORTANT: don’t throw (your current filter throws and Jetty turns it into 500) :contentReference[oaicite:3]{index=3} resp.resetBuffer(); resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); // Helpful for OAuth clients: resp.setHeader("WWW-Authenticate", - "Bearer realm=\"Veeam Integration\", error=\"" + esc(error) + "\", error_description=\"" + esc(desc) + "\""); + "Bearer realm=\"Veeam Integration\", error=\"" + DataUtil.jsonEscape(error) + + "\", error_description=\"" + DataUtil.jsonEscape(desc) + "\""); final String accept = req.getHeader("Accept"); final boolean wantsJson = accept != null && accept.toLowerCase().contains("application/json"); @@ -215,27 +209,12 @@ private static void unauthorized(HttpServletRequest req, HttpServletResponse res resp.setCharacterEncoding("UTF-8"); if (wantsJson) { resp.setContentType("application/json; charset=UTF-8"); - resp.getWriter().write("{\"error\":\"" + esc(error) + "\",\"error_description\":\"" + esc(desc) + "\"}"); + resp.getWriter().write("{\"error\":\"" + DataUtil.jsonEscape(error) + + "\",\"error_description\":\"" + DataUtil.jsonEscape(desc) + "\"}"); } else { resp.setContentType("text/html; charset=UTF-8"); resp.getWriter().write("ErrorUnauthorized"); } resp.getWriter().flush(); } - - private static String esc(String s) { - return s == null ? "" : s.replace("\\", "\\\\").replace("\"", "\\\""); - } - - private static boolean constantTimeEquals(String a, String b) { - if (a == null || b == null) return false; - return constantTimeEquals(a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8)); - } - - private static boolean constantTimeEquals(byte[] x, byte[] y) { - if (x.length != y.length) return false; - int r = 0; - for (int i = 0; i < x.length; i++) r |= x[i] ^ y[i]; - return r == 0; - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java index a402b88ab76c..3f1735952012 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java @@ -20,27 +20,30 @@ import java.io.IOException; import java.time.Instant; import java.util.HashMap; +import java.util.List; import java.util.Map; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; +import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.VeeamControlServlet; -import org.apache.cloudstack.veeam.filter.BearerOrBasicAuthFilter; +import org.apache.cloudstack.veeam.utils.JwtUtil; import org.apache.cloudstack.veeam.utils.Negotiation; +import org.apache.commons.lang3.StringUtils; import com.cloud.utils.component.ManagerBase; public class SsoService extends ManagerBase implements RouteHandler { private static final String BASE_ROUTE = "/sso"; private static final long DEFAULT_TTL_SECONDS = 3600; + public static final List REQUIRED_SCOPES = List.of("ovirt-app-admin", "ovirt-app-portal"); + public static final String HMAC_SECRET = "change-this-super-secret-key-change-this"; - // Replace with your real credential validation (CloudStack account, config, etc.) - private final PasswordAuthenticator authenticator = new StaticPasswordAuthenticator(); + @Inject + VeeamControlService veeamControlService; @Override public boolean canHandle(String method, String path) { @@ -48,7 +51,8 @@ public boolean canHandle(String method, String path) { } @Override - public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { + public void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, + VeeamControlServlet io) throws IOException { final String sanitizedPath = getSanitizedPath(path); if (sanitizedPath.equals(BASE_ROUTE + "/oauth/token")) { handleToken(req, resp, outFormat, io); @@ -62,54 +66,56 @@ protected void handleToken(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { if (!"POST".equalsIgnoreCase(req.getMethod())) { - // oVirt-like: 405 is fine; if you strictly want 400, change to SC_BAD_REQUEST resp.setHeader("Allow", "POST"); io.getWriter().write(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, - Map.of("error", "method_not_allowed", "message", "token endpoint requires POST"), outFormat); + Map.of("error", "method_not_allowed", + "message", "token endpoint requires POST"), outFormat); return; } - // OAuth password grant uses x-www-form-urlencoded. With servlet containers this usually populates getParameter(). final String grantType = trimToNull(req.getParameter("grant_type")); - final String scope = trimToNull(req.getParameter("scope")); // typically "ovirt-app-api" + final String scope = trimToNull(req.getParameter("scope")); final String username = trimToNull(req.getParameter("username")); final String password = trimToNull(req.getParameter("password")); if (grantType == null) { io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, - Map.of("error", "invalid_request", "error_description", "Missing parameter: grant_type"), outFormat); + Map.of("error", "invalid_request", + "error_description", "Missing parameter: grant_type"), outFormat); return; } if (!"password".equals(grantType)) { io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, - Map.of("error", "unsupported_grant_type", "error_description", "Only grant_type=password is supported"), outFormat); + Map.of("error", "unsupported_grant_type", + "error_description", "Only grant_type=password is supported"), outFormat); return; } if (username == null || password == null) { io.getWriter().write(resp, HttpServletResponse.SC_BAD_REQUEST, - Map.of("error", "invalid_request", "error_description", "Missing username/password"), outFormat); + Map.of("error", "invalid_request", + "error_description", "Missing username/password"), outFormat); return; } - if (!authenticator.authenticate(username, password)) { - // 401 for bad creds + if (!veeamControlService.validateCredentials(username, password)) { io.getWriter().write(resp, HttpServletResponse.SC_UNAUTHORIZED, - Map.of("error", "invalid_grant", "error_description", "Invalid credentials"), outFormat); + Map.of("error", "invalid_grant", + "error_description", "Invalid credentials"), outFormat); return; } - final String effectiveScope = (scope == null) ? "ovirt-app-api" : scope; + final String effectiveScope = (scope == null) ? StringUtils.join(REQUIRED_SCOPES, " ") : scope; final long ttl = DEFAULT_TTL_SECONDS; long nowMillis = Instant.now().toEpochMilli(); long expMillis = nowMillis + ttl * 1000L; final String token; try { - token = JwtUtil.issueHs256Jwt(BearerOrBasicAuthFilter.ISSUER, username, effectiveScope, ttl, - BearerOrBasicAuthFilter.HMAC_SECRET); + token = JwtUtil.issueHs256Jwt(username, effectiveScope, ttl, HMAC_SECRET); } catch (Exception e) { io.getWriter().write(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - Map.of("error", "server_error", "error_description", "Failed to issue token"), outFormat); + Map.of("error", "server_error", + "error_description", "Failed to issue token"), outFormat); return; } @@ -128,61 +134,4 @@ private static String trimToNull(String s) { s = s.trim(); return s.isEmpty() ? null : s; } - - // ---------- Minimal auth helpers (replace later) ---------- - - interface PasswordAuthenticator { - boolean authenticate(String username, String password); - } - - static final class StaticPasswordAuthenticator implements PasswordAuthenticator { - StaticPasswordAuthenticator() { - } - @Override - public boolean authenticate(String username, String password) { - return VeeamControlService.Username.value().equals(username) && - VeeamControlService.Password.value().equals(password); - } - } - - // ---------- Minimal JWT HS256 without extra libs ---------- - // (If you prefer Nimbus, I can convert this to nimbus-jose-jwt; this keeps dependencies tiny.) - - static final class JwtUtil { - static String issueHs256Jwt(String issuer, String subject, String scope, long ttlSeconds, String secret) throws Exception { - long now = Instant.now().getEpochSecond(); - long exp = now + ttlSeconds; - - String headerJson = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; - String payloadJson = - "{" - + "\"iss\":\"" + jsonEscape(issuer) + "\"," - + "\"sub\":\"" + jsonEscape(subject) + "\"," - + "\"scope\":\"" + jsonEscape(scope) + "\"," - + "\"iat\":" + now + "," - + "\"exp\":" + exp - + "}"; - - String header = b64Url(headerJson.getBytes("UTF-8")); - String payload = b64Url(payloadJson.getBytes("UTF-8")); - String signingInput = header + "." + payload; - - byte[] sig = hmacSha256(signingInput.getBytes("UTF-8"), secret.getBytes("UTF-8")); - return signingInput + "." + b64Url(sig); - } - - static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { - Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(key, "HmacSHA256")); - return mac.doFinal(data); - } - - static String b64Url(byte[] in) { - return java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(in); - } - - static String jsonEscape(String s) { - return s.replace("\\", "\\\\").replace("\"", "\\\""); - } - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/DataUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/DataUtil.java new file mode 100644 index 000000000000..9e0eef768d03 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/DataUtil.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class DataUtil { + + public static String b64Url(byte[] in) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(in); + } + + public static String jsonEscape(String s) { + return s == null ? "" : s.replace("\\", "\\\\").replace("\"", "\\\""); + } + + public static boolean constantTimeEquals(String a, String b) { + if (a == null || b == null) return false; + return constantTimeEquals(a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8)); + } + + public static boolean constantTimeEquals(byte[] x, byte[] y) { + if (x.length != y.length) return false; + int r = 0; + for (int i = 0; i < x.length; i++) r |= x[i] ^ y[i]; + return r == 0; + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java new file mode 100644 index 000000000000..c4438525c34d --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.utils; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +public class JwtUtil { + public static final String ALGORITHM = "HmacSHA256"; + public static final String ISSUER = "veeam-control"; + + public static String issueHs256Jwt(String subject, String scope, long ttlSeconds, String secret) throws Exception { + long now = Instant.now().getEpochSecond(); + long exp = now + ttlSeconds; + + String headerJson = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String payloadJson = + "{" + + "\"iss\":\"" + DataUtil.jsonEscape(ISSUER) + "\"," + + "\"sub\":\"" + DataUtil.jsonEscape(subject) + "\"," + + "\"scope\":\"" + DataUtil.jsonEscape(scope) + "\"," + + "\"iat\":" + now + "," + + "\"exp\":" + exp + + "}"; + + String header = DataUtil.b64Url(headerJson.getBytes(StandardCharsets.UTF_8)); + String payload = DataUtil.b64Url(payloadJson.getBytes(StandardCharsets.UTF_8)); + String signingInput = header + "." + payload; + + byte[] sig = hmacSha256(signingInput.getBytes(StandardCharsets.UTF_8), secret.getBytes(StandardCharsets.UTF_8)); + return signingInput + "." + DataUtil.b64Url(sig); + } + + public static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { + final Mac mac = Mac.getInstance(ALGORITHM); + mac.init(new SecretKeySpec(key, ALGORITHM)); + return mac.doFinal(data); + } +} \ No newline at end of file From 5907d6427a21b40fc069c56d4367fb6e17955dc2 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:04:19 +0530 Subject: [PATCH 069/129] Use the upper ceiling (in gb) for the volume size during restore --- .../org/apache/cloudstack/veeam/adapter/ServerAdapter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index e463c02cdb01..8fe47387b93d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1024,7 +1024,9 @@ public Disk createDisk(Disk request) { if (provisionedSizeInGb <= 0) { throw new InvalidParameterValueException("Provisioned size must be greater than zero"); } - provisionedSizeInGb = Math.max(1L, provisionedSizeInGb / (1024L * 1024L * 1024L)); + // round-up provisionedSizeInGb to the next whole GB + long GB = 1024L * 1024L * 1024L; + provisionedSizeInGb = Math.max(1L, (provisionedSizeInGb + GB - 1) / GB); Long initialSize = null; if (StringUtils.isNotBlank(request.getInitialSize())) { try { From 3e7268e457c32de67f0056db1cbb86c7ad94c081 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:05:57 +0530 Subject: [PATCH 070/129] modularize image server --- scripts/vm/hypervisor/kvm/image_server.py | 1565 +---------------- .../vm/hypervisor/kvm/imageserver/__init__.py | 33 + .../vm/hypervisor/kvm/imageserver/__main__.py | 20 + .../kvm/imageserver/backends/__init__.py | 36 + .../kvm/imageserver/backends/base.py | 148 ++ .../kvm/imageserver/backends/file.py | 123 ++ .../kvm/imageserver/backends/nbd.py | 476 +++++ .../hypervisor/kvm/imageserver/concurrency.py | 71 + .../vm/hypervisor/kvm/imageserver/config.py | 136 ++ .../hypervisor/kvm/imageserver/constants.py | 29 + .../vm/hypervisor/kvm/imageserver/handler.py | 842 +++++++++ .../vm/hypervisor/kvm/imageserver/server.py | 75 + scripts/vm/hypervisor/kvm/imageserver/util.py | 79 + 13 files changed, 2072 insertions(+), 1561 deletions(-) create mode 100644 scripts/vm/hypervisor/kvm/imageserver/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/__main__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/base.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/file.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/concurrency.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/config.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/constants.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/handler.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/server.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/util.py diff --git a/scripts/vm/hypervisor/kvm/image_server.py b/scripts/vm/hypervisor/kvm/image_server.py index 891bac5bf53b..c0436b4d2077 100644 --- a/scripts/vm/hypervisor/kvm/image_server.py +++ b/scripts/vm/hypervisor/kvm/image_server.py @@ -16,1570 +16,13 @@ # specific language governing permissions and limitations # under the License. -""" -POC "imageio-like" HTTP server backed by NBD over Unix socket or a local file. -Supports two backends (see config payload): -- nbd: proxy to an NBD server via Unix socket (socket path, export, export_bitmap); - supports range reads/writes (GET/PUT/PATCH), extents, zero, flush. PUT accepts - full upload or ranged upload (Content-Range). Concurrent PUTs on the same image - are serialized (blocking). -- file: read/write a local qcow2 (or raw) file path; full PUT only (no range - writes), GET with optional ranges, flush. - -How to run ----------- -- Install dependency: - dnf install python3-libnbd - or - apt install python3-libnbd - -- Run server: - createImageTransfer will start the server as a systemd service 'cloudstack-image-server' - -Example curl commands --------------------- -- OPTIONS: - curl -i -X OPTIONS http://127.0.0.1:54323/images/demo - -- GET full image: - curl -v http://127.0.0.1:54323/images/demo -o demo.img - -- GET a byte range: - curl -v -H "Range: bytes=0-1048575" http://127.0.0.1:54323/images/demo -o first_1MiB.bin - -- PUT full image (Content-Length must equal export size exactly). Optional ?flush=y|n: - curl -v -T demo.img http://127.0.0.1:54323/images/demo - curl -v -T demo.img "http://127.0.0.1:54323/images/demo?flush=y" - -- PUT ranged (NBD backend only). Content-Range: bytes start-end/* or bytes start-end/size - (server uses only start; length from Content-Length). Optional ?flush=y|n: - curl -v -X PUT -H "Content-Range: bytes 0-1048575/*" -H "Content-Length: 1048576" \ - --data-binary @chunk.bin http://127.0.0.1:54323/images/demo - curl -v -X PUT -H "Content-Range: bytes 1048576-2097151/52428800" -H "Content-Length: 1048576" \ - --data-binary @chunk2.bin "http://127.0.0.1:54323/images/demo?flush=n" - -- GET extents (zero/hole extents from NBD base:allocation): - curl -s http://127.0.0.1:54323/images/demo/extents | jq . - -- GET extents with dirty and zero (requires export_bitmap in config): - curl -s "http://127.0.0.1:54323/images/demo/extents?context=dirty" | jq . - -- POST flush: - curl -s -X POST http://127.0.0.1:54323/images/demo/flush | jq . - -- PATCH zero (zero a byte range; application/json body): - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "offset": 4096, "size": 8192}' \ - http://127.0.0.1:54323/images/demo - - Zero at offset 1 GiB, 4096 bytes, no flush: - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "offset": 1073741824, "size": 4096}' \ - http://127.0.0.1:54323/images/demo - - Zero entire disk and flush: - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "zero", "size": 107374182400, "flush": true}' \ - http://127.0.0.1:54323/images/demo - -- PATCH flush (flush data to storage; operates on entire image): - curl -k -X PATCH \ - -H "Content-Type: application/json" \ - --data-binary '{"op": "flush"}' \ - http://127.0.0.1:54323/images/demo - -- PATCH range (write binary body at byte range; Range + Content-Length required): - curl -v -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin \ - http://127.0.0.1:54323/images/demo -""" - -import argparse -import json -import logging import os -import re -import socket -import threading -import time -from http import HTTPStatus -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn -try: - from http.server import ThreadingHTTPServer -except ImportError: - # Python 3.6: ThreadingHTTPServer was added in 3.7 - class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - pass -from typing import Any, Dict, List, Optional, Set, Tuple -from urllib.parse import parse_qs -import nbd - -CHUNK_SIZE = 256 * 1024 # 256 KiB - -# NBD base:allocation flags (hole=1, zero=2; hole|zero=3) -_NBD_STATE_HOLE = 1 -_NBD_STATE_ZERO = 2 -# NBD qemu:dirty-bitmap flags (dirty=1) -_NBD_STATE_DIRTY = 1 - -# Concurrency limits across ALL images. -MAX_PARALLEL_READS = 8 -MAX_PARALLEL_WRITES = 1 - -_READ_SEM = threading.Semaphore(MAX_PARALLEL_READS) -_WRITE_SEM = threading.Semaphore(MAX_PARALLEL_WRITES) - -# In-memory per-image lock: single lock gates both read and write. -_IMAGE_LOCKS: Dict[str, threading.Lock] = {} -_IMAGE_LOCKS_GUARD = threading.Lock() - - -# Dynamic image_id(transferId) -> backend mapping: -# CloudStack writes a JSON file at /tmp/imagetransfer/ with: -# - NBD backend: {"backend": "nbd", "socket": "/tmp/imagetransfer/.sock", "export": "vda", "export_bitmap": "..."} -# - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} -# -# This server reads that file on-demand. -_CFG_DIR = "/tmp/imagetransfer" -_CFG_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {} -_CFG_CACHE_GUARD = threading.Lock() - - -def _json_bytes(obj: Any) -> bytes: - return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - - -def _merge_dirty_zero_extents( - allocation_extents: List[Tuple[int, int, bool]], - dirty_extents: List[Tuple[int, int, bool]], - size: int, -) -> List[Dict[str, Any]]: - """ - Merge allocation (start, length, zero) and dirty (start, length, dirty) extents - into a single list of {start, length, dirty, zero} with unified boundaries. - """ - boundaries: Set[int] = {0, size} - for start, length, _ in allocation_extents: - boundaries.add(start) - boundaries.add(start + length) - for start, length, _ in dirty_extents: - boundaries.add(start) - boundaries.add(start + length) - sorted_boundaries = sorted(boundaries) - - def lookup( - extents: List[Tuple[int, int, bool]], offset: int, default: bool - ) -> bool: - for start, length, flag in extents: - if start <= offset < start + length: - return flag - return default - - result: List[Dict[str, Any]] = [] - for i in range(len(sorted_boundaries) - 1): - a, b = sorted_boundaries[i], sorted_boundaries[i + 1] - if a >= b: - continue - result.append( - { - "start": a, - "length": b - a, - "dirty": lookup(dirty_extents, a, False), - "zero": lookup(allocation_extents, a, False), - } - ) - return result - - -def _is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: - """True if extents is the single-extent fallback (dirty=false, zero=false).""" - return ( - len(extents) == 1 - and extents[0].get("dirty") is False - and extents[0].get("zero") is False - ) - - -def _get_image_lock(image_id: str) -> threading.Lock: - with _IMAGE_LOCKS_GUARD: - lock = _IMAGE_LOCKS.get(image_id) - if lock is None: - lock = threading.Lock() - _IMAGE_LOCKS[image_id] = lock - return lock - - -def _now_s() -> float: - return time.monotonic() - - -def _safe_transfer_id(image_id: str) -> Optional[str]: - """ - Only allow a single filename component to avoid path traversal. - We intentionally keep validation simple: reject anything containing '/' or '\\'. - """ - if not image_id: - return None - if image_id != os.path.basename(image_id): - return None - if "/" in image_id or "\\" in image_id: - return None - if image_id in (".", ".."): - return None - return image_id - - -def _load_image_cfg(image_id: str) -> Optional[Dict[str, Any]]: - safe_id = _safe_transfer_id(image_id) - if safe_id is None: - return None - - cfg_path = os.path.join(_CFG_DIR, safe_id) - try: - st = os.stat(cfg_path) - except FileNotFoundError: - return None - except OSError as e: - logging.error("cfg stat failed image_id=%s err=%r", image_id, e) - return None - - with _CFG_CACHE_GUARD: - cached = _CFG_CACHE.get(safe_id) - if cached is not None: - cached_mtime, cached_cfg = cached - # Use cached config if the file hasn't changed. - if float(st.st_mtime) == float(cached_mtime): - return cached_cfg - - try: - with open(cfg_path, "rb") as f: - raw = f.read(4096) - except OSError as e: - logging.error("cfg read failed image_id=%s err=%r", image_id, e) - return None - - try: - obj = json.loads(raw.decode("utf-8")) - except Exception as e: - logging.error("cfg parse failed image_id=%s err=%r", image_id, e) - return None - - if not isinstance(obj, dict): - logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) - return None - - backend = obj.get("backend") - if backend is None: - backend = "nbd" - if not isinstance(backend, str): - logging.error("cfg invalid backend type image_id=%s", image_id) - return None - backend = backend.lower() - if backend not in ("nbd", "file"): - logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) - return None - - if backend == "file": - file_path = obj.get("file") - if not isinstance(file_path, str) or not file_path.strip(): - logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) - return None - cfg = {"backend": "file", "file": file_path.strip()} - else: - socket_path = obj.get("socket") - export = obj.get("export") - export_bitmap = obj.get("export_bitmap") - if not isinstance(socket_path, str) or not socket_path.strip(): - logging.error("cfg missing/invalid socket path for nbd backend image_id=%s", image_id) - return None - socket_path = socket_path.strip() - if export is not None and (not isinstance(export, str) or not export): - logging.error("cfg missing/invalid export image_id=%s", image_id) - return None - cfg = { - "backend": "nbd", - "socket": socket_path, - "export": export, - "export_bitmap": export_bitmap, - } - - with _CFG_CACHE_GUARD: - _CFG_CACHE[safe_id] = (float(st.st_mtime), cfg) - return cfg - - -class _NbdConn: - """ - Small helper to connect to NBD over a Unix socket. - Opens a fresh handle per request, per POC requirements. - """ - - def __init__( - self, - socket_path: str, - export: Optional[str], - need_block_status: bool = False, - extra_meta_contexts: Optional[List[str]] = None, - ): - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._sock.connect(socket_path) - self._nbd = nbd.NBD() - - # Select export name if supported/needed. - if export and hasattr(self._nbd, "set_export_name"): - self._nbd.set_export_name(export) - - # Request meta contexts before connect (for block status / dirty bitmap). - if need_block_status and hasattr(self._nbd, "add_meta_context"): - for ctx in ["base:allocation"] + (extra_meta_contexts or []): - try: - self._nbd.add_meta_context(ctx) - except Exception as e: - logging.warning("add_meta_context %r failed: %r", ctx, e) - - self._connect_existing_socket(self._sock) - - def _connect_existing_socket(self, sock: socket.socket) -> None: - # Requirement: attach libnbd to an existing socket / FD (no qemu-nbd). - # libnbd python API varies slightly by version, so try common options. - last_err: Optional[BaseException] = None - if hasattr(self._nbd, "connect_socket"): - try: - self._nbd.connect_socket(sock) - return - except Exception as e: # pragma: no cover (depends on binding) - last_err = e - try: - self._nbd.connect_socket(sock.fileno()) - return - except Exception as e2: # pragma: no cover - last_err = e2 - if hasattr(self._nbd, "connect_fd"): - try: - self._nbd.connect_fd(sock.fileno()) - return - except Exception as e: # pragma: no cover - last_err = e - raise RuntimeError( - "Unable to connect libnbd using existing socket/fd; " - f"binding missing connect_socket/connect_fd or call failed: {last_err!r}" - ) - - def size(self) -> int: - return int(self._nbd.get_size()) - - def get_capabilities(self) -> Dict[str, bool]: - """ - Query NBD export capabilities (read_only, can_flush, can_zero) from the - server handshake. Returns dict with keys read_only, can_flush, can_zero. - Uses getattr for binding name variations (is_read_only/get_read_only, etc.). - """ - out: Dict[str, bool] = { - "read_only": True, - "can_flush": False, - "can_zero": False, - } - for name, keys in [ - ("read_only", ("is_read_only", "get_read_only")), - ("can_flush", ("can_flush", "get_can_flush")), - ("can_zero", ("can_zero", "get_can_zero")), - ]: - for attr in keys: - if hasattr(self._nbd, attr): - try: - val = getattr(self._nbd, attr)() - out[name] = bool(val) - except Exception: - pass - break - return out - - def pread(self, length: int, offset: int) -> bytes: - # Expected signature: pread(length, offset) - try: - return self._nbd.pread(length, offset) - except TypeError: # pragma: no cover (binding differences) - return self._nbd.pread(offset, length) - - def pwrite(self, buf: bytes, offset: int) -> None: - # Expected signature: pwrite(buf, offset) - try: - self._nbd.pwrite(buf, offset) - except TypeError: # pragma: no cover (binding differences) - self._nbd.pwrite(offset, buf) - - def pzero(self, offset: int, size: int) -> None: - """ - Zero a byte range. Uses NBD WRITE_ZEROES when available (efficient/punch hole), - otherwise falls back to writing zero bytes via pwrite. - """ - if size <= 0: - return - # Try libnbd pwrite_zeros / zero; argument order varies by binding. - for name in ("pwrite_zeros", "zero"): - if not hasattr(self._nbd, name): - continue - fn = getattr(self._nbd, name) - try: - fn(size, offset) - return - except TypeError: - try: - fn(offset, size) - return - except TypeError: - pass - # Fallback: write zeros in chunks. - remaining = size - pos = offset - zero_buf = b"\x00" * min(CHUNK_SIZE, size) - while remaining > 0: - chunk = min(len(zero_buf), remaining) - self.pwrite(zero_buf[:chunk], pos) - pos += chunk - remaining -= chunk - - def flush(self) -> None: - if hasattr(self._nbd, "flush"): - self._nbd.flush() - return - if hasattr(self._nbd, "fsync"): - self._nbd.fsync() - return - raise RuntimeError("libnbd binding has no flush/fsync method") - - def get_allocation_extents(self) -> List[Dict[str, Any]]: - """ - Query base:allocation and return all extents (allocated and hole/zero) - as [{"start": ..., "length": ..., "zero": bool}, ...]. - Fallback when block status unavailable: one extent with zero=False. - """ - size = self.size() - if size == 0: - return [] - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - return [{"start": 0, "length": size, "zero": False}] - if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( - "base:allocation" - ): - return [{"start": 0, "length": size, "zero": False}] - - allocation_extents: List[Dict[str, Any]] = [] - chunk = min(size, 64 * 1024 * 1024) - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - if len(args) < 3: - return 0 - metacontext, off, entries = args[0], args[1], args[2] - if metacontext != "base:allocation" or entries is None: - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 - allocation_extents.append( - {"start": current, "length": length, "zero": zero} - ) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return [{"start": 0, "length": size, "zero": False}] - try: - while offset < size: - count = min(chunk, size - offset) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.warning("get_allocation_extents block_status failed: %r", e) - return [{"start": 0, "length": size, "zero": False}] - if not allocation_extents: - return [{"start": 0, "length": size, "zero": False}] - return allocation_extents - - def get_extents_dirty_and_zero( - self, dirty_bitmap_context: str - ) -> List[Dict[str, Any]]: - """ - Query block status for base:allocation and qemu:dirty-bitmap:, - merge boundaries, and return extents with dirty and zero flags. - Format: [{"start": ..., "length": ..., "dirty": bool, "zero": bool}, ...]. - """ - size = self.size() - if size == 0: - return [] - if not hasattr(self._nbd, "block_status") and not hasattr( - self._nbd, "block_status_64" - ): - return self._fallback_dirty_zero_extents(size) - if hasattr(self._nbd, "can_meta_context"): - if not self._nbd.can_meta_context("base:allocation"): - return self._fallback_dirty_zero_extents(size) - if not self._nbd.can_meta_context(dirty_bitmap_context): - logging.warning( - "dirty bitmap context %r not negotiated", dirty_bitmap_context - ) - return self._fallback_dirty_zero_extents(size) - - allocation_extents: List[Tuple[int, int, bool]] = [] # (start, length, zero) - dirty_extents: List[Tuple[int, int, bool]] = [] # (start, length, dirty) - chunk = min(size, 64 * 1024 * 1024) - offset = 0 - - def extent_cb(*args: Any, **kwargs: Any) -> int: - if len(args) < 3: - return 0 - metacontext, off, entries = args[0], args[1], args[2] - if entries is None or not hasattr(entries, "__iter__"): - return 0 - current = off - try: - flat = list(entries) - for i in range(0, len(flat), 2): - if i + 1 >= len(flat): - break - length = int(flat[i]) - flags = int(flat[i + 1]) - if metacontext == "base:allocation": - zero = (flags & (_NBD_STATE_HOLE | _NBD_STATE_ZERO)) != 0 - allocation_extents.append((current, length, zero)) - elif metacontext == dirty_bitmap_context: - dirty = (flags & _NBD_STATE_DIRTY) != 0 - dirty_extents.append((current, length, dirty)) - current += length - except (TypeError, ValueError, IndexError): - pass - return 0 - - block_status_fn = getattr( - self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) - ) - if block_status_fn is None: - return self._fallback_dirty_zero_extents(size) - try: - while offset < size: - count = min(chunk, size - offset) - try: - block_status_fn(count, offset, extent_cb) - except TypeError: - block_status_fn(offset, count, extent_cb) - offset += count - except Exception as e: - logging.warning("get_extents_dirty_and_zero block_status failed: %r", e) - return self._fallback_dirty_zero_extents(size) - return _merge_dirty_zero_extents(allocation_extents, dirty_extents, size) - - def _fallback_dirty_zero_extents(self, size: int) -> List[Dict[str, Any]]: - """One extent: whole image, dirty=false, zero=false when bitmap unavailable.""" - return [{"start": 0, "length": size, "dirty": False, "zero": False}] - - def close(self) -> None: - # Best-effort; bindings may differ. - try: - if hasattr(self._nbd, "shutdown"): - self._nbd.shutdown() - except Exception: - pass - try: - if hasattr(self._nbd, "close"): - self._nbd.close() - except Exception: - pass - try: - self._sock.close() - except Exception: - pass - - def __enter__(self) -> "_NbdConn": - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - -class Handler(BaseHTTPRequestHandler): - server_version = "imageio-poc/0.1" - server_protocol = "HTTP/1.1" - - # Accept both "bytes start-end/*" and "bytes start-end/size"; we only use start. - _CONTENT_RANGE_RE = re.compile(r"^bytes\s+(\d+)-(\d+)/(?:\*|\d+)$") - - # Keep BaseHTTPRequestHandler from printing noisy default logs - def log_message(self, fmt: str, *args: Any) -> None: - logging.info("%s - - %s", self.address_string(), fmt % args) - - def _send_imageio_headers( - self, allowed_methods: Optional[str] = None - ) -> None: - # Include these headers for compatibility with the imageio contract. - if allowed_methods is None: - allowed_methods = "GET, PUT, OPTIONS" - self.send_header("Access-Control-Allow-Methods", allowed_methods) - self.send_header("Accept-Ranges", "bytes") - - def _send_json( - self, - status: int, - obj: Any, - allowed_methods: Optional[str] = None, - ) -> None: - body = _json_bytes(obj) - self.send_response(status) - self._send_imageio_headers(allowed_methods) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - try: - self.wfile.write(body) - except BrokenPipeError: - pass - - def _send_error_json(self, status: int, message: str) -> None: - self._send_json(status, {"error": message}) - - def _send_range_not_satisfiable(self, size: int) -> None: - # RFC 7233: reply with Content-Range: bytes */ - self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) - self._send_imageio_headers() - self.send_header("Content-Type", "application/json") - self.send_header("Content-Range", f"bytes */{size}") - body = _json_bytes({"error": "range not satisfiable"}) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - try: - self.wfile.write(body) - except BrokenPipeError: - pass - - def _parse_single_range(self, range_header: str, size: int) -> Tuple[int, int]: - """ - Parse a single HTTP byte range (RFC 7233) and return (start, end_inclusive). - - Supported: - - Range: bytes=START-END - - Range: bytes=START- - - Range: bytes=-SUFFIX - - Raises ValueError for invalid headers. Caller handles 416 vs 400. - """ - if size < 0: - raise ValueError("invalid size") - if not range_header: - raise ValueError("empty Range") - if "," in range_header: - raise ValueError("multiple ranges not supported") - - prefix = "bytes=" - if not range_header.startswith(prefix): - raise ValueError("only bytes ranges supported") - spec = range_header[len(prefix) :].strip() - if "-" not in spec: - raise ValueError("invalid bytes range") - - left, right = spec.split("-", 1) - left = left.strip() - right = right.strip() - - if left == "": - # Suffix range: last N bytes. - if right == "": - raise ValueError("invalid suffix range") - try: - suffix_len = int(right, 10) - except ValueError as e: - raise ValueError("invalid suffix length") from e - if suffix_len <= 0: - raise ValueError("invalid suffix length") - if size == 0: - # Nothing to serve - raise ValueError("unsatisfiable") - if suffix_len >= size: - return 0, size - 1 - return size - suffix_len, size - 1 - - # START is present - try: - start = int(left, 10) - except ValueError as e: - raise ValueError("invalid range start") from e - if start < 0: - raise ValueError("invalid range start") - if start >= size: - raise ValueError("unsatisfiable") - - if right == "": - # START- - return start, size - 1 - - try: - end = int(right, 10) - except ValueError as e: - raise ValueError("invalid range end") from e - if end < start: - raise ValueError("unsatisfiable") - if end >= size: - end = size - 1 - return start, end - - def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: - # Returns (image_id, tail) where tail is: - # None => /images/{id} - # "extents" => /images/{id}/extents - # "flush" => /images/{id}/flush - path = self.path.split("?", 1)[0] - parts = [p for p in path.split("/") if p] - if len(parts) < 2 or parts[0] != "images": - return None, None - image_id = parts[1] - tail = parts[2] if len(parts) >= 3 else None - if len(parts) > 3: - return None, None - return image_id, tail - - def _parse_content_range(self, header: str) -> Tuple[int, int]: - """ - Parse Content-Range header "bytes start-end/*" or "bytes start-end/size" - and return (start, end_inclusive). Raises ValueError on invalid input. - """ - if not header: - raise ValueError("empty Content-Range") - m = self._CONTENT_RANGE_RE.match(header.strip()) - if not m: - raise ValueError("invalid Content-Range") - start_s, end_s = m.groups() - start = int(start_s, 10) - end = int(end_s, 10) - if start < 0 or end < start: - raise ValueError("invalid Content-Range range") - return start, end - - def _parse_query(self) -> Dict[str, List[str]]: - """Parse query string from self.path into a dict of name -> list of values.""" - if "?" not in self.path: - return {} - query = self.path.split("?", 1)[1] - return parse_qs(query, keep_blank_values=True) - - def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: - return _load_image_cfg(image_id) - - def _is_file_backend(self, cfg: Dict[str, Any]) -> bool: - return cfg.get("backend") == "file" - - def do_OPTIONS(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - if self._is_file_backend(cfg): - # File backend: full PUT only, no range writes; GET with ranges allowed; flush supported. - allowed_methods = "GET, PUT, POST, OPTIONS" - features = ["flush"] - max_writers = MAX_PARALLEL_WRITES - response = { - "unix_socket": None, - "features": features, - "max_readers": MAX_PARALLEL_READS, - "max_writers": max_writers, - } - self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - return - # Query NBD backend for capabilities (like nbdinfo); fall back to config. - read_only = True - can_flush = False - can_zero = False - try: - with _NbdConn( - cfg["socket"], - cfg.get("export"), - ) as conn: - caps = conn.get_capabilities() - read_only = caps["read_only"] - can_flush = caps["can_flush"] - can_zero = caps["can_zero"] - except Exception as e: - logging.warning("OPTIONS: could not query NBD capabilities: %r", e) - read_only = bool(cfg.get("read_only")) - if not read_only: - can_flush = True - can_zero = True - # Report options for this image from NBD: read-only => no PUT; only advertise supported features. - if read_only: - allowed_methods = "GET, OPTIONS" - features = ["extents"] - max_writers = 0 - else: - # PATCH: JSON (zero/flush) and Range+binary (write byte range). - allowed_methods = "GET, PUT, PATCH, OPTIONS" - features = ["extents"] - if can_zero: - features.append("zero") - if can_flush: - features.append("flush") - max_writers = MAX_PARALLEL_WRITES if not read_only else 0 - response = { - "unix_socket": None, # Not used in this implementation - "features": features, - "max_readers": MAX_PARALLEL_READS, - "max_writers": max_writers, - } - self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - - def do_GET(self) -> None: - image_id, tail = self._parse_route() - if image_id is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - if tail == "extents": - if self._is_file_backend(cfg): - self._send_error_json( - HTTPStatus.BAD_REQUEST, "extents not supported for file backend" - ) - return - query = self._parse_query() - context = (query.get("context") or [None])[0] - self._handle_get_extents(image_id, cfg, context=context) - return - if tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - range_header = self.headers.get("Range") - self._handle_get_image(image_id, cfg, range_header) - - def do_PUT(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - # For PUT we only support Content-Range (for NBD backend); Range is rejected. - if self.headers.get("Range") is not None: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "Range header not supported for PUT; use Content-Range or PATCH", - ) - return - - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - - # Optional flush=y|n query parameter. - query = self._parse_query() - flush_param = (query.get("flush") or ["n"])[0].lower() - flush = flush_param in ("y", "yes", "true", "1") - - content_range_hdr = self.headers.get("Content-Range") - if content_range_hdr is not None: - if self._is_file_backend(cfg): - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "Content-Range PUT not supported for file backend; use full PUT", - ) - return - self._handle_put_range(image_id, cfg, content_range_hdr, content_length, flush) - return - - # No Content-Range: full image PUT. - self._handle_put_image(image_id, cfg, content_length, flush) - - def do_POST(self) -> None: - image_id, tail = self._parse_route() - if image_id is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - - if tail == "flush": - self._handle_post_flush(image_id, cfg) - return - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - - def do_PATCH(self) -> None: - image_id, tail = self._parse_route() - if image_id is None or tail is not None: - self._send_error_json(HTTPStatus.NOT_FOUND, "not found") - return - - cfg = self._image_cfg(image_id) - if cfg is None: - self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") - return - if self._is_file_backend(cfg): - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "range writes and PATCH not supported for file backend; use PUT for full upload", - ) - return - - content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() - range_header = self.headers.get("Range") - - # Binary PATCH: Range + body writes bytes at that range (e.g. curl -X PATCH -H "Range: bytes=0-1048576" --data-binary @chunk.bin). - if range_header is not None and content_type != "application/json": - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") - return - self._handle_patch_range(image_id, cfg, range_header, content_length) - return - - # JSON PATCH: application/json with op (zero, flush). - if content_type != "application/json": - self._send_error_json( - HTTPStatus.UNSUPPORTED_MEDIA_TYPE, - "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", - ) - return - - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length <= 0 or content_length > 64 * 1024: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - - body = self.rfile.read(content_length) - if len(body) != content_length: - self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") - return - - try: - payload = json.loads(body.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError) as e: - self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") - return - - if not isinstance(payload, dict): - self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") - return - - op = payload.get("op") - if op == "flush": - # Flush entire image; offset and size are ignored (per spec). - self._handle_post_flush(image_id, cfg) - return - if op != "zero": - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "unsupported op; only \"zero\" and \"flush\" are supported", - ) - return - - try: - size = int(payload.get("size")) - except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") - return - if size <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") - return - - offset = payload.get("offset") - if offset is None: - offset = 0 - else: - try: - offset = int(offset) - except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") - return - if offset < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") - return - - flush = bool(payload.get("flush", False)) - - self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) - - def _handle_get_image( - self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] - ) -> None: - if not _READ_SEM.acquire(blocking=False): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") - return - - start = _now_s() - bytes_sent = 0 - try: - logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") - if self._is_file_backend(cfg): - file_path = cfg["file"] - try: - size = os.path.getsize(file_path) - except OSError as e: - logging.error("GET file size error image_id=%s path=%s err=%r", image_id, file_path, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "failed to access file") - return - start_off = 0 - end_off_incl = size - 1 if size > 0 else -1 - status = HTTPStatus.OK - content_length = size - if range_header is not None: - try: - start_off, end_off_incl = self._parse_single_range(range_header, size) - except ValueError as e: - if str(e) == "unsatisfiable": - self._send_range_not_satisfiable(size) - return - if "unsatisfiable" in str(e): - self._send_range_not_satisfiable(size) - return - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") - return - status = HTTPStatus.PARTIAL_CONTENT - content_length = (end_off_incl - start_off) + 1 - - self.send_response(status) - self._send_imageio_headers() - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(content_length)) - if status == HTTPStatus.PARTIAL_CONTENT: - self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") - self.end_headers() - - offset = start_off - end_excl = end_off_incl + 1 - with open(file_path, "rb") as f: - f.seek(offset) - while offset < end_excl: - to_read = min(CHUNK_SIZE, end_excl - offset) - data = f.read(to_read) - if not data: - break - try: - self.wfile.write(data) - except BrokenPipeError: - logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) - break - offset += len(data) - bytes_sent += len(data) - else: - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - size = conn.size() - - start_off = 0 - end_off_incl = size - 1 if size > 0 else -1 - status = HTTPStatus.OK - content_length = size - if range_header is not None: - try: - start_off, end_off_incl = self._parse_single_range(range_header, size) - except ValueError as e: - if str(e) == "unsatisfiable": - self._send_range_not_satisfiable(size) - return - if "unsatisfiable" in str(e): - self._send_range_not_satisfiable(size) - return - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") - return - status = HTTPStatus.PARTIAL_CONTENT - content_length = (end_off_incl - start_off) + 1 - - self.send_response(status) - self._send_imageio_headers() - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(content_length)) - if status == HTTPStatus.PARTIAL_CONTENT: - self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") - self.end_headers() - - offset = start_off - end_excl = end_off_incl + 1 - while offset < end_excl: - to_read = min(CHUNK_SIZE, end_excl - offset) - data = conn.pread(to_read, offset) - if not data: - raise RuntimeError("backend returned empty read") - try: - self.wfile.write(data) - except BrokenPipeError: - logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) - break - offset += len(data) - bytes_sent += len(data) - except Exception as e: - # If headers already sent, we can't return JSON reliably; just log. - logging.error("GET error image_id=%s err=%r", image_id, e) - try: - if not self.wfile.closed: - self.close_connection = True - except Exception: - pass - finally: - _READ_SEM.release() - dur = _now_s() - start - logging.info( - "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur - ) - - def _handle_put_image( - self, image_id: str, cfg: Dict[str, Any], content_length: int, flush: bool - ) -> None: - lock = _get_image_lock(image_id) - # Block until we can write this image - lock.acquire() - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - bytes_written = 0 - try: - logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) - if self._is_file_backend(cfg): - file_path = cfg["file"] - remaining = content_length - with open(file_path, "wb") as f: - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {bytes_written} bytes", - ) - return - f.write(chunk) - bytes_written += len(chunk) - remaining -= len(chunk) - if flush: - f.flush() - os.fsync(f.fileno()) - self._send_json( - HTTPStatus.OK, - {"ok": True, "bytes_written": bytes_written, "flushed": flush}, - ) - else: - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - offset = 0 - remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {offset} bytes", - ) - return - conn.pwrite(chunk, offset) - offset += len(chunk) - remaining -= len(chunk) - bytes_written += len(chunk) - - if flush: - conn.flush() - self._send_json( - HTTPStatus.OK, - {"ok": True, "bytes_written": bytes_written, "flushed": flush}, - ) - except Exception as e: - logging.error("PUT error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur - ) - - def _write_range_nbd( - self, - image_id: str, - cfg: Dict[str, Any], - start_off: int, - content_length: int, - ) -> Tuple[int, bool]: - """ - Low-level helper: write request body to NBD backend starting at start_off. - The length is taken from Content-Length. Returns (bytes_written, ok). - If ok is False, an error response was already sent. - """ - bytes_written = 0 - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - image_size = conn.size() - if start_off >= image_size: - self._send_range_not_satisfiable(image_size) - return 0, False - - # Clamp to image size: we do not allow writes beyond end of image. - max_len = image_size - start_off - if content_length > max_len: - self._send_range_not_satisfiable(image_size) - return 0, False - - offset = start_off - remaining = content_length - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"request body ended early at {bytes_written} bytes", - ) - return bytes_written, False - conn.pwrite(chunk, offset) - n = len(chunk) - offset += n - remaining -= n - bytes_written += n - - return bytes_written, True - - def _handle_get_extents( - self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None - ) -> None: - # context=dirty: return extents with dirty and zero from base:allocation + bitmap. - # Otherwise: return zero/hole extents from base:allocation only. - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - start = _now_s() - try: - logging.info("EXTENTS start image_id=%s context=%s", image_id, context) - if context == "dirty": - export_bitmap = cfg.get("export_bitmap") - if not export_bitmap: - # Fallback: same structure as zero extents but dirty=true for all ranges - with _NbdConn( - cfg["socket"], - cfg.get("export"), - need_block_status=True, - ) as conn: - allocation = conn.get_allocation_extents() - extents = [ - {"start": e["start"], "length": e["length"], "dirty": True, "zero": e["zero"]} - for e in allocation - ] - else: - dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}" - extra_contexts: List[str] = [dirty_bitmap_ctx] - with _NbdConn( - cfg["socket"], - cfg.get("export"), - need_block_status=True, - extra_meta_contexts=extra_contexts, - ) as conn: - extents = conn.get_extents_dirty_and_zero(dirty_bitmap_ctx) - # When bitmap not actually available, same fallback: zero structure + dirty=true - if _is_fallback_dirty_response(extents): - with _NbdConn( - cfg["socket"], - cfg.get("export"), - need_block_status=True, - ) as conn: - allocation = conn.get_allocation_extents() - extents = [ - { - "start": e["start"], - "length": e["length"], - "dirty": True, - "zero": e["zero"], - } - for e in allocation - ] - else: - with _NbdConn( - cfg["socket"], - cfg.get("export"), - need_block_status=True, - ) as conn: - extents = conn.get_allocation_extents() - self._send_json(HTTPStatus.OK, extents) - except Exception as e: - logging.error("EXTENTS error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - lock.release() - dur = _now_s() - start - logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - start = _now_s() - try: - logging.info("FLUSH start image_id=%s", image_id) - if self._is_file_backend(cfg): - file_path = cfg["file"] - with open(file_path, "rb") as f: - f.flush() - os.fsync(f.fileno()) - self._send_json(HTTPStatus.OK, {"ok": True}) - else: - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - conn.flush() - self._send_json(HTTPStatus.OK, {"ok": True}) - except Exception as e: - logging.error("FLUSH error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - lock.release() - dur = _now_s() - start - logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_patch_zero( - self, - image_id: str, - cfg: Dict[str, Any], - offset: int, - size: int, - flush: bool, - ) -> None: - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - try: - logging.info( - "PATCH zero start image_id=%s offset=%d size=%d flush=%s", - image_id, offset, size, flush, - ) - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - image_size = conn.size() - if offset >= image_size: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "offset must be less than image size", - ) - return - zero_size = min(size, image_size - offset) - conn.pzero(offset, zero_size) - if flush: - conn.flush() - self._send_json(HTTPStatus.OK, {"ok": True}) - except Exception as e: - logging.error("PATCH zero error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) - - def _handle_patch_range( - self, - image_id: str, - cfg: Dict[str, Any], - range_header: str, - content_length: int, - ) -> None: - """Write request body to the image at the byte range from Range header.""" - lock = _get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - bytes_written = 0 - try: - logging.info( - "PATCH range start image_id=%s range=%s content_length=%d", - image_id, range_header, content_length, - ) - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - image_size = conn.size() - try: - start_off, end_inclusive = self._parse_single_range( - range_header, image_size - ) - except ValueError as e: - if "unsatisfiable" in str(e).lower(): - self._send_range_not_satisfiable(image_size) - else: - self._send_error_json( - HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" - ) - return - expected_len = end_inclusive - start_off + 1 - if content_length != expected_len: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"Content-Length ({content_length}) must equal range length ({expected_len})", - ) - return - bytes_written, ok = self._write_range_nbd(image_id, cfg, start_off, content_length) - if not ok: - return - self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) - except Exception as e: - logging.error("PATCH range error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "PATCH range end image_id=%s bytes=%d duration_s=%.3f", - image_id, bytes_written, dur, - ) - - def _handle_put_range( - self, - image_id: str, - cfg: Dict[str, Any], - content_range: str, - content_length: int, - flush: bool, - ) -> None: - """Handle PUT with Content-Range: bytes start-end/* for NBD backend.""" - lock = _get_image_lock(image_id) - # Block until we can write this image. - lock.acquire() - - if not _WRITE_SEM.acquire(blocking=False): - lock.release() - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - - start = _now_s() - bytes_written = 0 - try: - logging.info( - "PUT range start image_id=%s Content-Range=%s content_length=%d flush=%s", - image_id, - content_range, - content_length, - flush, - ) - try: - start_off, end_inclusive = self._parse_content_range(content_range) - except ValueError as e: - self._send_error_json( - HTTPStatus.BAD_REQUEST, f"invalid Content-Range header: {e}" - ) - return - - # Per contract we only use the start byte from Content-Range; - # length comes from Content-Length. - bytes_written, ok = self._write_range_nbd(image_id, cfg, start_off, content_length) - if not ok: - return - - if flush: - with _NbdConn(cfg["socket"], cfg.get("export")) as conn: - conn.flush() - - self._send_json( - HTTPStatus.OK, - {"ok": True, "bytes_written": bytes_written, "flushed": flush}, - ) - except Exception as e: - logging.error("PUT range error image_id=%s err=%r", image_id, e) - self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") - finally: - _WRITE_SEM.release() - lock.release() - dur = _now_s() - start - logging.info( - "PUT range end image_id=%s bytes=%d duration_s=%.3f flush=%s", - image_id, - bytes_written, - dur, - flush, - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description="POC imageio-like HTTP server backed by NBD") - parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") - parser.add_argument("--port", type=int, default=54323, help="Port to listen on") - args = parser.parse_args() - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - ) +import sys - addr = (args.listen, args.port) - httpd = ThreadingHTTPServer(addr, Handler) - logging.info("listening on http://%s:%d", args.listen, args.port) - logging.info("image configs are read from %s/", _CFG_DIR) - httpd.serve_forever() +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from imageserver.server import main if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/scripts/vm/hypervisor/kvm/imageserver/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/__init__.py new file mode 100644 index 000000000000..5e033f5d527f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/__init__.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +CloudStack image server — HTTP server backed by NBD over Unix socket or a local file. + +Supports two backends (configured per-transfer via JSON config): +- nbd: proxy to an NBD server via Unix socket; supports range reads/writes + (GET/PUT/PATCH), extents, zero, flush. +- file: read/write a local qcow2/raw file; full PUT only, GET with optional + ranges, flush. + +Usage:: + + # As a module + python -m imageserver --listen 127.0.0.1 --port 54323 + + # Or via the systemd service started by createImageTransfer +""" diff --git a/scripts/vm/hypervisor/kvm/imageserver/__main__.py b/scripts/vm/hypervisor/kvm/imageserver/__main__.py new file mode 100644 index 000000000000..e64bd5f65205 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/__main__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .server import main + +main() diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/backends/__init__.py new file mode 100644 index 000000000000..36080b4cbe73 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/__init__.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Dict + +from .base import BackendSession, ImageBackend +from .file import FileBackend +from .nbd import NbdBackend + +__all__ = ["BackendSession", "ImageBackend", "FileBackend", "NbdBackend", "create_backend"] + + +def create_backend(cfg: Dict[str, Any]) -> ImageBackend: + """Factory: build the correct ImageBackend from a transfer config dict.""" + backend_type = cfg.get("backend", "nbd") + if backend_type == "file": + return FileBackend(cfg["file"]) + return NbdBackend( + cfg["socket"], + export=cfg.get("export"), + export_bitmap=cfg.get("export_bitmap"), + ) diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/base.py b/scripts/vm/hypervisor/kvm/imageserver/backends/base.py new file mode 100644 index 000000000000..b081640e2d38 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/base.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Any, Dict, List + + +class BackendSession(ABC): + """ + A session that holds an open connection/file handle for the duration of + an operation (e.g. a full GET streaming read). Use as a context manager. + """ + + @abstractmethod + def size(self) -> int: + """Return the image size in bytes.""" + ... + + @abstractmethod + def read(self, offset: int, length: int) -> bytes: + """ + Read *length* bytes starting at *offset*. + + For NBD backends, raises RuntimeError if the server returns empty data. + For file backends, returns empty bytes at EOF. + """ + ... + + @abstractmethod + def close(self) -> None: + """Release the underlying connection or file handle.""" + ... + + def __enter__(self) -> "BackendSession": + return self + + def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + self.close() + + +class ImageBackend(ABC): + """ + Abstract base class for image storage backends. + + Each backend (NBD, file, etc.) implements this interface so the HTTP handler + can operate uniformly without backend-specific branching. + """ + + @property + @abstractmethod + def supports_extents(self) -> bool: + """Whether this backend supports querying allocation/dirty extents.""" + ... + + @property + @abstractmethod + def supports_range_write(self) -> bool: + """Whether this backend supports writing at arbitrary byte offsets.""" + ... + + @abstractmethod + def size(self) -> int: + """Return the image size in bytes.""" + ... + + @abstractmethod + def read(self, offset: int, length: int) -> bytes: + """Read *length* bytes starting at *offset*.""" + ... + + @abstractmethod + def write(self, data: bytes, offset: int) -> None: + """Write *data* at *offset*.""" + ... + + @abstractmethod + def write_full(self, stream, content_length: int, flush: bool) -> int: + """ + Consume *content_length* bytes from *stream* and write the full image. + Returns bytes written. Raises on short read. + """ + ... + + @abstractmethod + def flush(self) -> None: + """Flush pending data to stable storage.""" + ... + + @abstractmethod + def zero(self, offset: int, length: int) -> None: + """Zero *length* bytes starting at *offset*.""" + ... + + @abstractmethod + def get_capabilities(self) -> Dict[str, bool]: + """ + Return backend capabilities dict with keys: + read_only, can_flush, can_zero. + """ + ... + + @abstractmethod + def get_allocation_extents(self) -> List[Dict[str, Any]]: + """ + Return allocation extents as [{"start": int, "length": int, "zero": bool}, ...]. + """ + ... + + @abstractmethod + def get_dirty_extents(self, dirty_bitmap_context: str) -> List[Dict[str, Any]]: + """ + Return merged dirty+zero extents as + [{"start": int, "length": int, "dirty": bool, "zero": bool}, ...]. + """ + ... + + @abstractmethod + def open_session(self) -> BackendSession: + """ + Open a session that holds a single connection/file handle for the + duration of a streaming operation (e.g. GET). + """ + ... + + @abstractmethod + def close(self) -> None: + """Release any resources held by this backend.""" + ... + + def __enter__(self) -> "ImageBackend": + return self + + def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + self.close() diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/file.py b/scripts/vm/hypervisor/kvm/imageserver/backends/file.py new file mode 100644 index 000000000000..9e55bf21fdef --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/file.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from io import BufferedReader +from typing import Any, Dict, List, Optional + +from ..constants import CHUNK_SIZE +from .base import BackendSession, ImageBackend + + +class FileSession(BackendSession): + """ + Holds a single file handle open for the duration of a streaming read. + Returns empty bytes at EOF (file semantics). + """ + + def __init__(self, path: str): + self._path = path + self._fh: Optional[BufferedReader] = open(path, "rb") + self._size = os.path.getsize(path) + + def size(self) -> int: + return self._size + + def read(self, offset: int, length: int) -> bytes: + if self._fh is None: + raise RuntimeError("session is closed") + self._fh.seek(offset) + return self._fh.read(length) + + def close(self) -> None: + if self._fh is not None: + self._fh.close() + self._fh = None + + +class FileBackend(ImageBackend): + """ + ImageBackend implementation backed by a local file (qcow2 or raw). + Supports full read/write and flush. Does not support extents or range writes. + """ + + def __init__(self, file_path: str): + self._path = file_path + + @property + def supports_extents(self) -> bool: + return False + + @property + def supports_range_write(self) -> bool: + return False + + def size(self) -> int: + return os.path.getsize(self._path) + + def read(self, offset: int, length: int) -> bytes: + with open(self._path, "rb") as f: + f.seek(offset) + return f.read(length) + + def write(self, data: bytes, offset: int) -> None: + raise NotImplementedError("file backend does not support range writes") + + def write_full(self, stream: Any, content_length: int, flush: bool) -> int: + bytes_written = 0 + remaining = content_length + with open(self._path, "wb") as f: + while remaining > 0: + chunk = stream.read(min(CHUNK_SIZE, remaining)) + if not chunk: + raise IOError( + f"request body ended early at {bytes_written} bytes" + ) + f.write(chunk) + bytes_written += len(chunk) + remaining -= len(chunk) + if flush: + f.flush() + os.fsync(f.fileno()) + return bytes_written + + def flush(self) -> None: + with open(self._path, "rb") as f: + f.flush() + os.fsync(f.fileno()) + + def zero(self, offset: int, length: int) -> None: + raise NotImplementedError("file backend does not support zero") + + def get_capabilities(self) -> Dict[str, bool]: + return { + "read_only": False, + "can_flush": True, + "can_zero": False, + } + + def get_allocation_extents(self) -> List[Dict[str, Any]]: + raise NotImplementedError("file backend does not support extents") + + def get_dirty_extents(self, dirty_bitmap_context: str) -> List[Dict[str, Any]]: + raise NotImplementedError("file backend does not support extents") + + def open_session(self) -> FileSession: + return FileSession(self._path) + + def close(self) -> None: + pass diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py new file mode 100644 index 000000000000..ed6d3ac6ed7a --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py @@ -0,0 +1,476 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import socket +from typing import Any, Dict, List, Optional, Tuple + +import nbd + +from ..constants import CHUNK_SIZE, NBD_STATE_DIRTY, NBD_STATE_HOLE, NBD_STATE_ZERO +from ..util import merge_dirty_zero_extents +from .base import BackendSession, ImageBackend + + +class NbdConnection: + """ + Low-level helper to connect to an NBD server over a Unix socket. + Opens a fresh handle per connection. + """ + + def __init__( + self, + socket_path: str, + export: Optional[str], + need_block_status: bool = False, + extra_meta_contexts: Optional[List[str]] = None, + ): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(socket_path) + self._nbd = nbd.NBD() + + if export and hasattr(self._nbd, "set_export_name"): + self._nbd.set_export_name(export) + + if need_block_status and hasattr(self._nbd, "add_meta_context"): + for ctx in ["base:allocation"] + (extra_meta_contexts or []): + try: + self._nbd.add_meta_context(ctx) + except Exception as e: + logging.warning("add_meta_context %r failed: %r", ctx, e) + + self._connect_existing_socket(self._sock) + + def _connect_existing_socket(self, sock: socket.socket) -> None: + last_err: Optional[BaseException] = None + if hasattr(self._nbd, "connect_socket"): + try: + self._nbd.connect_socket(sock) + return + except Exception as e: + last_err = e + try: + self._nbd.connect_socket(sock.fileno()) + return + except Exception as e2: + last_err = e2 + if hasattr(self._nbd, "connect_fd"): + try: + self._nbd.connect_fd(sock.fileno()) + return + except Exception as e: + last_err = e + raise RuntimeError( + "Unable to connect libnbd using existing socket/fd; " + f"binding missing connect_socket/connect_fd or call failed: {last_err!r}" + ) + + def size(self) -> int: + return int(self._nbd.get_size()) + + def get_capabilities(self) -> Dict[str, bool]: + """ + Query NBD export capabilities (read_only, can_flush, can_zero) from the + server handshake. Uses getattr for binding name variations. + """ + out: Dict[str, bool] = { + "read_only": True, + "can_flush": False, + "can_zero": False, + } + for name, keys in [ + ("read_only", ("is_read_only", "get_read_only")), + ("can_flush", ("can_flush", "get_can_flush")), + ("can_zero", ("can_zero", "get_can_zero")), + ]: + for attr in keys: + if hasattr(self._nbd, attr): + try: + val = getattr(self._nbd, attr)() + out[name] = bool(val) + except Exception: + pass + break + return out + + def pread(self, length: int, offset: int) -> bytes: + try: + return self._nbd.pread(length, offset) + except TypeError: + return self._nbd.pread(offset, length) + + def pwrite(self, buf: bytes, offset: int) -> None: + try: + self._nbd.pwrite(buf, offset) + except TypeError: + self._nbd.pwrite(offset, buf) + + def pzero(self, offset: int, size: int) -> None: + """ + Zero a byte range. Uses NBD WRITE_ZEROES when available, + otherwise falls back to writing zero bytes via pwrite. + """ + if size <= 0: + return + for fn_name in ("pwrite_zeros", "zero"): + if not hasattr(self._nbd, fn_name): + continue + fn = getattr(self._nbd, fn_name) + try: + fn(size, offset) + return + except TypeError: + try: + fn(offset, size) + return + except TypeError: + pass + remaining = size + pos = offset + zero_buf = b"\x00" * min(CHUNK_SIZE, size) + while remaining > 0: + chunk = min(len(zero_buf), remaining) + self.pwrite(zero_buf[:chunk], pos) + pos += chunk + remaining -= chunk + + def flush(self) -> None: + if hasattr(self._nbd, "flush"): + self._nbd.flush() + return + if hasattr(self._nbd, "fsync"): + self._nbd.fsync() + return + raise RuntimeError("libnbd binding has no flush/fsync method") + + def get_allocation_extents(self) -> List[Dict[str, Any]]: + """ + Query base:allocation and return all extents as + [{"start": ..., "length": ..., "zero": bool}, ...]. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return [{"start": 0, "length": size, "zero": False}] + if hasattr(self._nbd, "can_meta_context") and not self._nbd.can_meta_context( + "base:allocation" + ): + return [{"start": 0, "length": size, "zero": False}] + + allocation_extents: List[Dict[str, Any]] = [] + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if metacontext != "base:allocation" or entries is None: + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + zero = (flags & (NBD_STATE_HOLE | NBD_STATE_ZERO)) != 0 + allocation_extents.append( + {"start": current, "length": length, "zero": zero} + ) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return [{"start": 0, "length": size, "zero": False}] + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_allocation_extents block_status failed: %r", e) + return [{"start": 0, "length": size, "zero": False}] + if not allocation_extents: + return [{"start": 0, "length": size, "zero": False}] + return allocation_extents + + def get_extents_dirty_and_zero( + self, dirty_bitmap_context: str + ) -> List[Dict[str, Any]]: + """ + Query block status for base:allocation and a dirty bitmap context, + merge boundaries, and return extents with dirty and zero flags. + """ + size = self.size() + if size == 0: + return [] + if not hasattr(self._nbd, "block_status") and not hasattr( + self._nbd, "block_status_64" + ): + return self._fallback_dirty_zero_extents(size) + if hasattr(self._nbd, "can_meta_context"): + if not self._nbd.can_meta_context("base:allocation"): + return self._fallback_dirty_zero_extents(size) + if not self._nbd.can_meta_context(dirty_bitmap_context): + logging.warning( + "dirty bitmap context %r not negotiated", dirty_bitmap_context + ) + return self._fallback_dirty_zero_extents(size) + + allocation_extents: List[Tuple[int, int, bool]] = [] + dirty_extents: List[Tuple[int, int, bool]] = [] + chunk = min(size, 64 * 1024 * 1024) + offset = 0 + + def extent_cb(*args: Any, **kwargs: Any) -> int: + if len(args) < 3: + return 0 + metacontext, off, entries = args[0], args[1], args[2] + if entries is None or not hasattr(entries, "__iter__"): + return 0 + current = off + try: + flat = list(entries) + for i in range(0, len(flat), 2): + if i + 1 >= len(flat): + break + length = int(flat[i]) + flags = int(flat[i + 1]) + if metacontext == "base:allocation": + zero = (flags & (NBD_STATE_HOLE | NBD_STATE_ZERO)) != 0 + allocation_extents.append((current, length, zero)) + elif metacontext == dirty_bitmap_context: + dirty = (flags & NBD_STATE_DIRTY) != 0 + dirty_extents.append((current, length, dirty)) + current += length + except (TypeError, ValueError, IndexError): + pass + return 0 + + block_status_fn = getattr( + self._nbd, "block_status_64", getattr(self._nbd, "block_status", None) + ) + if block_status_fn is None: + return self._fallback_dirty_zero_extents(size) + try: + while offset < size: + count = min(chunk, size - offset) + try: + block_status_fn(count, offset, extent_cb) + except TypeError: + block_status_fn(offset, count, extent_cb) + offset += count + except Exception as e: + logging.warning("get_extents_dirty_and_zero block_status failed: %r", e) + return self._fallback_dirty_zero_extents(size) + return merge_dirty_zero_extents(allocation_extents, dirty_extents, size) + + @staticmethod + def _fallback_dirty_zero_extents(size: int) -> List[Dict[str, Any]]: + return [{"start": 0, "length": size, "dirty": False, "zero": False}] + + def close(self) -> None: + try: + if hasattr(self._nbd, "shutdown"): + self._nbd.shutdown() + except Exception: + pass + try: + if hasattr(self._nbd, "close"): + self._nbd.close() + except Exception: + pass + try: + self._sock.close() + except Exception: + pass + + def __enter__(self) -> "NbdConnection": + return self + + def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + self.close() + + +class NbdSession(BackendSession): + """ + Holds a single NbdConnection open for the duration of a streaming operation. + Raises RuntimeError if pread returns empty data (NBD should never do this). + """ + + def __init__(self, conn: NbdConnection): + self._conn = conn + + def size(self) -> int: + return self._conn.size() + + def read(self, offset: int, length: int) -> bytes: + data = self._conn.pread(length, offset) + if not data: + raise RuntimeError("backend returned empty read") + return data + + def close(self) -> None: + self._conn.close() + + +class NbdBackend(ImageBackend): + """ + ImageBackend implementation that proxies to an NBD server via Unix socket. + Each public method opens a fresh NbdConnection (per the original design). + """ + + def __init__( + self, + socket_path: str, + export: Optional[str] = None, + export_bitmap: Optional[str] = None, + ): + self._socket_path = socket_path + self._export = export + self._export_bitmap = export_bitmap + + @property + def supports_extents(self) -> bool: + return True + + @property + def supports_range_write(self) -> bool: + return True + + @property + def export_bitmap(self) -> Optional[str]: + return self._export_bitmap + + def _connect( + self, + need_block_status: bool = False, + extra_meta_contexts: Optional[List[str]] = None, + ) -> NbdConnection: + return NbdConnection( + self._socket_path, + self._export, + need_block_status=need_block_status, + extra_meta_contexts=extra_meta_contexts, + ) + + def size(self) -> int: + with self._connect() as conn: + return conn.size() + + def read(self, offset: int, length: int) -> bytes: + with self._connect() as conn: + return conn.pread(length, offset) + + def write(self, data: bytes, offset: int) -> None: + with self._connect() as conn: + conn.pwrite(data, offset) + + def write_full(self, stream: Any, content_length: int, flush: bool) -> int: + bytes_written = 0 + with self._connect() as conn: + offset = 0 + remaining = content_length + while remaining > 0: + chunk = stream.read(min(CHUNK_SIZE, remaining)) + if not chunk: + raise IOError( + f"request body ended early at {offset} bytes" + ) + conn.pwrite(chunk, offset) + offset += len(chunk) + remaining -= len(chunk) + bytes_written += len(chunk) + if flush: + conn.flush() + return bytes_written + + def write_range(self, stream: Any, start_off: int, content_length: int) -> int: + """ + Write *content_length* bytes from *stream* to the image starting at *start_off*. + Returns bytes written. Raises ValueError if offset/length is out of bounds. + """ + bytes_written = 0 + with self._connect() as conn: + image_size = conn.size() + if start_off >= image_size: + raise ValueError(f"offset {start_off} >= image size {image_size}") + max_len = image_size - start_off + if content_length > max_len: + raise ValueError( + f"content_length {content_length} exceeds available space {max_len}" + ) + offset = start_off + remaining = content_length + while remaining > 0: + chunk = stream.read(min(CHUNK_SIZE, remaining)) + if not chunk: + raise IOError( + f"request body ended early at {bytes_written} bytes" + ) + conn.pwrite(chunk, offset) + n = len(chunk) + offset += n + remaining -= n + bytes_written += n + return bytes_written + + def flush(self) -> None: + with self._connect() as conn: + conn.flush() + + def zero(self, offset: int, length: int) -> None: + with self._connect() as conn: + image_size = conn.size() + if offset >= image_size: + raise ValueError("offset must be less than image size") + zero_size = min(length, image_size - offset) + conn.pzero(offset, zero_size) + + def get_capabilities(self) -> Dict[str, bool]: + with self._connect() as conn: + return conn.get_capabilities() + + def get_allocation_extents(self) -> List[Dict[str, Any]]: + with self._connect(need_block_status=True) as conn: + return conn.get_allocation_extents() + + def get_dirty_extents(self, dirty_bitmap_context: str) -> List[Dict[str, Any]]: + extra_contexts: List[str] = [dirty_bitmap_context] + with self._connect( + need_block_status=True, extra_meta_contexts=extra_contexts + ) as conn: + return conn.get_extents_dirty_and_zero(dirty_bitmap_context) + + def open_session(self) -> NbdSession: + return NbdSession(self._connect()) + + def close(self) -> None: + pass diff --git a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py new file mode 100644 index 000000000000..a446786224d8 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import threading +from typing import Dict, NamedTuple + + +class _ImageState(NamedTuple): + read_sem: threading.Semaphore + write_sem: threading.Semaphore + lock: threading.Lock + + +class ConcurrencyManager: + """ + Manages per-image read/write semaphores and per-image mutual-exclusion locks. + + Each image_id gets its own independent pool of read slots (default 8) + and write slots (default 1), so concurrent transfers to different images + do not contend with each other. + + The per-image lock serialises operations that must not overlap on the + same image (e.g. flush while writing, extents while writing). + """ + + def __init__(self, max_reads: int, max_writes: int): + self._max_reads = max_reads + self._max_writes = max_writes + self._images: Dict[str, _ImageState] = {} + self._guard = threading.Lock() + + def _state_for(self, image_id: str) -> _ImageState: + with self._guard: + state = self._images.get(image_id) + if state is None: + state = _ImageState( + read_sem=threading.Semaphore(self._max_reads), + write_sem=threading.Semaphore(self._max_writes), + lock=threading.Lock(), + ) + self._images[image_id] = state + return state + + def acquire_read(self, image_id: str, blocking: bool = False) -> bool: + return self._state_for(image_id).read_sem.acquire(blocking=blocking) + + def release_read(self, image_id: str) -> None: + self._state_for(image_id).read_sem.release() + + def acquire_write(self, image_id: str, blocking: bool = False) -> bool: + return self._state_for(image_id).write_sem.acquire(blocking=blocking) + + def release_write(self, image_id: str) -> None: + self._state_for(image_id).write_sem.release() + + def get_image_lock(self, image_id: str) -> threading.Lock: + return self._state_for(image_id).lock diff --git a/scripts/vm/hypervisor/kvm/imageserver/config.py b/scripts/vm/hypervisor/kvm/imageserver/config.py new file mode 100644 index 000000000000..cc0107cce9d0 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/config.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import logging +import os +import threading +from typing import Any, Dict, Optional, Tuple + +from .constants import CFG_DIR + + +def safe_transfer_id(image_id: str) -> Optional[str]: + """ + Only allow a single filename component to avoid path traversal. + Rejects anything containing '/' or '\\'. + """ + if not image_id: + return None + if image_id != os.path.basename(image_id): + return None + if "/" in image_id or "\\" in image_id: + return None + if image_id in (".", ".."): + return None + return image_id + + +class TransferConfigLoader: + """ + Loads and caches per-image transfer configuration from JSON files. + + CloudStack writes a JSON file at / with: + - NBD backend: {"backend": "nbd", "socket": "...", "export": "vda", "export_bitmap": "..."} + - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} + """ + + def __init__(self, cfg_dir: str = CFG_DIR): + self._cfg_dir = cfg_dir + self._cache: Dict[str, Tuple[float, Dict[str, Any]]] = {} + self._cache_guard = threading.Lock() + + @property + def cfg_dir(self) -> str: + return self._cfg_dir + + def load(self, image_id: str) -> Optional[Dict[str, Any]]: + safe_id = safe_transfer_id(image_id) + if safe_id is None: + return None + + cfg_path = os.path.join(self._cfg_dir, safe_id) + try: + st = os.stat(cfg_path) + except FileNotFoundError: + return None + except OSError as e: + logging.error("cfg stat failed image_id=%s err=%r", image_id, e) + return None + + with self._cache_guard: + cached = self._cache.get(safe_id) + if cached is not None: + cached_mtime, cached_cfg = cached + if float(st.st_mtime) == float(cached_mtime): + return cached_cfg + + try: + with open(cfg_path, "rb") as f: + raw = f.read(4096) + except OSError as e: + logging.error("cfg read failed image_id=%s err=%r", image_id, e) + return None + + try: + obj = json.loads(raw.decode("utf-8")) + except Exception as e: + logging.error("cfg parse failed image_id=%s err=%r", image_id, e) + return None + + if not isinstance(obj, dict): + logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) + return None + + backend = obj.get("backend") + if backend is None: + backend = "nbd" + if not isinstance(backend, str): + logging.error("cfg invalid backend type image_id=%s", image_id) + return None + backend = backend.lower() + if backend not in ("nbd", "file"): + logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) + return None + + if backend == "file": + file_path = obj.get("file") + if not isinstance(file_path, str) or not file_path.strip(): + logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) + return None + cfg: Dict[str, Any] = {"backend": "file", "file": file_path.strip()} + else: + socket_path = obj.get("socket") + export = obj.get("export") + export_bitmap = obj.get("export_bitmap") + if not isinstance(socket_path, str) or not socket_path.strip(): + logging.error("cfg missing/invalid socket path for nbd backend image_id=%s", image_id) + return None + socket_path = socket_path.strip() + if export is not None and (not isinstance(export, str) or not export): + logging.error("cfg missing/invalid export image_id=%s", image_id) + return None + cfg = { + "backend": "nbd", + "socket": socket_path, + "export": export, + "export_bitmap": export_bitmap, + } + + with self._cache_guard: + self._cache[safe_id] = (float(st.st_mtime), cfg) + return cfg diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py new file mode 100644 index 000000000000..6836f5798070 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CHUNK_SIZE = 256 * 1024 # 256 KiB + +# NBD base:allocation flags (hole=1, zero=2; hole|zero=3) +NBD_STATE_HOLE = 1 +NBD_STATE_ZERO = 2 +# NBD qemu:dirty-bitmap flags (dirty=1) +NBD_STATE_DIRTY = 1 + +MAX_PARALLEL_READS = 8 +MAX_PARALLEL_WRITES = 1 + +CFG_DIR = "/tmp/imagetransfer" diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py new file mode 100644 index 000000000000..8d894f9b0c58 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -0,0 +1,842 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import logging +import re +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qs + +from .backends import NbdBackend, create_backend +from .concurrency import ConcurrencyManager +from .config import TransferConfigLoader +from .constants import CHUNK_SIZE, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES +from .util import is_fallback_dirty_response, json_bytes, now_s + + +class Handler(BaseHTTPRequestHandler): + """ + HTTP request handler for the image server. + + Routing, HTTP parsing, and response formatting live here. + All backend I/O is delegated to ImageBackend implementations via the + create_backend() factory. + + Class-level attributes _concurrency and _config_loader are injected + by the server at startup (see server.py / make_handler()). + """ + + server_version = "cloudstack-image-server/1.0" + server_protocol = "HTTP/1.1" + + _concurrency: ConcurrencyManager + _config_loader: TransferConfigLoader + + _CONTENT_RANGE_RE = re.compile(r"^bytes\s+(\d+)-(\d+)/(?:\*|\d+)$") + + def log_message(self, fmt: str, *args: Any) -> None: + logging.info("%s - - %s", self.address_string(), fmt % args) + + # ------------------------------------------------------------------ + # Response helpers + # ------------------------------------------------------------------ + + def _send_imageio_headers( + self, allowed_methods: Optional[str] = None + ) -> None: + if allowed_methods is None: + allowed_methods = "GET, PUT, OPTIONS" + self.send_header("Access-Control-Allow-Methods", allowed_methods) + self.send_header("Accept-Ranges", "bytes") + + def _send_json( + self, + status: int, + obj: Any, + allowed_methods: Optional[str] = None, + ) -> None: + body = json_bytes(obj) + self.send_response(status) + self._send_imageio_headers(allowed_methods) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + def _send_error_json(self, status: int, message: str) -> None: + self._send_json(status, {"error": message}) + + def _send_range_not_satisfiable(self, size: int) -> None: + self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) + self._send_imageio_headers() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Range", f"bytes */{size}") + body = json_bytes({"error": "range not satisfiable"}) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + # ------------------------------------------------------------------ + # Parsing helpers + # ------------------------------------------------------------------ + + def _parse_single_range(self, range_header: str, size: int) -> Tuple[int, int]: + """ + Parse a single HTTP byte range (RFC 7233) and return (start, end_inclusive). + Raises ValueError for invalid headers. + """ + if size < 0: + raise ValueError("invalid size") + if not range_header: + raise ValueError("empty Range") + if "," in range_header: + raise ValueError("multiple ranges not supported") + + prefix = "bytes=" + if not range_header.startswith(prefix): + raise ValueError("only bytes ranges supported") + spec = range_header[len(prefix):].strip() + if "-" not in spec: + raise ValueError("invalid bytes range") + + left, right = spec.split("-", 1) + left = left.strip() + right = right.strip() + + if left == "": + if right == "": + raise ValueError("invalid suffix range") + try: + suffix_len = int(right, 10) + except ValueError as e: + raise ValueError("invalid suffix length") from e + if suffix_len <= 0: + raise ValueError("invalid suffix length") + if size == 0: + raise ValueError("unsatisfiable") + if suffix_len >= size: + return 0, size - 1 + return size - suffix_len, size - 1 + + try: + start = int(left, 10) + except ValueError as e: + raise ValueError("invalid range start") from e + if start < 0: + raise ValueError("invalid range start") + if start >= size: + raise ValueError("unsatisfiable") + + if right == "": + return start, size - 1 + + try: + end = int(right, 10) + except ValueError as e: + raise ValueError("invalid range end") from e + if end < start: + raise ValueError("unsatisfiable") + if end >= size: + end = size - 1 + return start, end + + def _parse_route(self) -> Tuple[Optional[str], Optional[str]]: + path = self.path.split("?", 1)[0] + parts = [p for p in path.split("/") if p] + if len(parts) < 2 or parts[0] != "images": + return None, None + image_id = parts[1] + tail = parts[2] if len(parts) >= 3 else None + if len(parts) > 3: + return None, None + return image_id, tail + + def _parse_content_range(self, header: str) -> Tuple[int, int]: + """ + Parse Content-Range header "bytes start-end/*" or "bytes start-end/size". + Returns (start, end_inclusive). + """ + if not header: + raise ValueError("empty Content-Range") + m = self._CONTENT_RANGE_RE.match(header.strip()) + if not m: + raise ValueError("invalid Content-Range") + start_s, end_s = m.groups() + start = int(start_s, 10) + end = int(end_s, 10) + if start < 0 or end < start: + raise ValueError("invalid Content-Range range") + return start, end + + def _parse_query(self) -> Dict[str, List[str]]: + if "?" not in self.path: + return {} + query = self.path.split("?", 1)[1] + return parse_qs(query, keep_blank_values=True) + + def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: + return self._config_loader.load(image_id) + + # ------------------------------------------------------------------ + # HTTP verb dispatchers + # ------------------------------------------------------------------ + + def do_OPTIONS(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + backend = create_backend(cfg) + try: + if not backend.supports_extents: + allowed_methods = "GET, PUT, POST, OPTIONS" + features = ["flush"] + response = { + "unix_socket": None, + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": MAX_PARALLEL_WRITES, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + return + + read_only = True + can_flush = False + can_zero = False + try: + caps = backend.get_capabilities() + read_only = caps["read_only"] + can_flush = caps["can_flush"] + can_zero = caps["can_zero"] + except Exception as e: + logging.warning("OPTIONS: could not query backend capabilities: %r", e) + read_only = bool(cfg.get("read_only")) + if not read_only: + can_flush = True + can_zero = True + + if read_only: + allowed_methods = "GET, OPTIONS" + features = ["extents"] + max_writers = 0 + else: + allowed_methods = "GET, PUT, PATCH, OPTIONS" + features = ["extents"] + if can_zero: + features.append("zero") + if can_flush: + features.append("flush") + max_writers = MAX_PARALLEL_WRITES + + response = { + "unix_socket": None, + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": max_writers, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + finally: + backend.close() + + def do_GET(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "extents": + backend = create_backend(cfg) + try: + if not backend.supports_extents: + self._send_error_json( + HTTPStatus.BAD_REQUEST, "extents not supported for file backend" + ) + return + finally: + backend.close() + query = self._parse_query() + context = (query.get("context") or [None])[0] + self._handle_get_extents(image_id, cfg, context=context) + return + if tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + range_header = self.headers.get("Range") + self._handle_get_image(image_id, cfg, range_header) + + def do_PUT(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if self.headers.get("Range") is not None: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "Range header not supported for PUT; use Content-Range or PATCH", + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + query = self._parse_query() + flush_param = (query.get("flush") or ["n"])[0].lower() + flush = flush_param in ("y", "yes", "true", "1") + + content_range_hdr = self.headers.get("Content-Range") + if content_range_hdr is not None: + backend = create_backend(cfg) + try: + if not backend.supports_range_write: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "Content-Range PUT not supported for file backend; use full PUT", + ) + return + finally: + backend.close() + self._handle_put_range(image_id, cfg, content_range_hdr, content_length, flush) + return + + self._handle_put_image(image_id, cfg, content_length, flush) + + def do_POST(self) -> None: + image_id, tail = self._parse_route() + if image_id is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + if tail == "flush": + self._handle_post_flush(image_id, cfg) + return + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + + def do_PATCH(self) -> None: + image_id, tail = self._parse_route() + if image_id is None or tail is not None: + self._send_error_json(HTTPStatus.NOT_FOUND, "not found") + return + + cfg = self._image_cfg(image_id) + if cfg is None: + self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") + return + + backend = create_backend(cfg) + try: + if not backend.supports_range_write: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "range writes and PATCH not supported for file backend; use PUT for full upload", + ) + return + finally: + backend.close() + + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + range_header = self.headers.get("Range") + + if range_header is not None and content_type != "application/json": + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") + return + self._handle_patch_range(image_id, cfg, range_header, content_length) + return + + if content_type != "application/json": + self._send_error_json( + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", + ) + return + + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0 or content_length > 64 * 1024: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + + body = self.rfile.read(content_length) + if len(body) != content_length: + self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") + return + + try: + payload = json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") + return + + if not isinstance(payload, dict): + self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") + return + + op = payload.get("op") + if op == "flush": + self._handle_post_flush(image_id, cfg) + return + if op != "zero": + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "unsupported op; only \"zero\" and \"flush\" are supported", + ) + return + + try: + size = int(payload.get("size")) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") + return + if size <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") + return + + offset = payload.get("offset") + if offset is None: + offset = 0 + else: + try: + offset = int(offset) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") + return + if offset < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") + return + + flush = bool(payload.get("flush", False)) + self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) + + # ------------------------------------------------------------------ + # Operation handlers + # ------------------------------------------------------------------ + + def _handle_get_image( + self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] + ) -> None: + if not self._concurrency.acquire_read(image_id): + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") + return + + start = now_s() + bytes_sent = 0 + try: + logging.info("GET start image_id=%s range=%s", image_id, range_header or "-") + backend = create_backend(cfg) + session = None + try: + session = backend.open_session() + size = session.size() + except OSError as e: + logging.error("GET size error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "failed to access image") + if session is not None: + session.close() + backend.close() + return + + try: + start_off = 0 + end_off_incl = size - 1 if size > 0 else -1 + status = HTTPStatus.OK + content_length = size + if range_header is not None: + try: + start_off, end_off_incl = self._parse_single_range(range_header, size) + except ValueError as e: + if "unsatisfiable" in str(e): + self._send_range_not_satisfiable(size) + return + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid Range header") + return + status = HTTPStatus.PARTIAL_CONTENT + content_length = (end_off_incl - start_off) + 1 + + self.send_response(status) + self._send_imageio_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(content_length)) + if status == HTTPStatus.PARTIAL_CONTENT: + self.send_header("Content-Range", f"bytes {start_off}-{end_off_incl}/{size}") + self.end_headers() + + offset = start_off + end_excl = end_off_incl + 1 + while offset < end_excl: + to_read = min(CHUNK_SIZE, end_excl - offset) + data = session.read(offset, to_read) + if not data: + break + try: + self.wfile.write(data) + except BrokenPipeError: + logging.info("GET client disconnected image_id=%s at=%d", image_id, offset) + break + offset += len(data) + bytes_sent += len(data) + finally: + session.close() + backend.close() + except Exception as e: + logging.error("GET error image_id=%s err=%r", image_id, e) + try: + if not self.wfile.closed: + self.close_connection = True + except Exception: + pass + finally: + self._concurrency.release_read(image_id) + dur = now_s() - start + logging.info( + "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur + ) + + def _handle_put_image( + self, image_id: str, cfg: Dict[str, Any], content_length: int, flush: bool + ) -> None: + lock = self._concurrency.get_image_lock(image_id) + lock.acquire() + + if not self._concurrency.acquire_write(image_id): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = now_s() + bytes_written = 0 + try: + logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) + backend = create_backend(cfg) + try: + bytes_written = backend.write_full(self.rfile, content_length, flush) + self._send_json( + HTTPStatus.OK, + {"ok": True, "bytes_written": bytes_written, "flushed": flush}, + ) + except IOError as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, str(e)) + finally: + backend.close() + except Exception as e: + logging.error("PUT error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + self._concurrency.release_write(image_id) + lock.release() + dur = now_s() - start + logging.info( + "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur + ) + + def _handle_put_range( + self, + image_id: str, + cfg: Dict[str, Any], + content_range: str, + content_length: int, + flush: bool, + ) -> None: + lock = self._concurrency.get_image_lock(image_id) + lock.acquire() + + if not self._concurrency.acquire_write(image_id): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = now_s() + bytes_written = 0 + try: + logging.info( + "PUT range start image_id=%s Content-Range=%s content_length=%d flush=%s", + image_id, content_range, content_length, flush, + ) + try: + start_off, _end_inclusive = self._parse_content_range(content_range) + except ValueError as e: + self._send_error_json( + HTTPStatus.BAD_REQUEST, f"invalid Content-Range header: {e}" + ) + return + + backend = create_backend(cfg) + try: + nbd_backend: NbdBackend = backend # type: ignore[assignment] + bytes_written = nbd_backend.write_range(self.rfile, start_off, content_length) + if flush: + nbd_backend.flush() + self._send_json( + HTTPStatus.OK, + {"ok": True, "bytes_written": bytes_written, "flushed": flush}, + ) + except ValueError: + image_size = backend.size() + self._send_range_not_satisfiable(image_size) + except IOError as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, str(e)) + finally: + backend.close() + except Exception as e: + logging.error("PUT range error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + self._concurrency.release_write(image_id) + lock.release() + dur = now_s() - start + logging.info( + "PUT range end image_id=%s bytes=%d duration_s=%.3f flush=%s", + image_id, bytes_written, dur, flush, + ) + + def _handle_get_extents( + self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None + ) -> None: + lock = self._concurrency.get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = now_s() + try: + logging.info("EXTENTS start image_id=%s context=%s", image_id, context) + backend = create_backend(cfg) + try: + if context == "dirty": + nbd_backend: NbdBackend = backend # type: ignore[assignment] + export_bitmap = nbd_backend.export_bitmap + if not export_bitmap: + allocation = nbd_backend.get_allocation_extents() + extents: List[Dict[str, Any]] = [ + {"start": e["start"], "length": e["length"], "dirty": True, "zero": e["zero"]} + for e in allocation + ] + else: + dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}" + extents = nbd_backend.get_dirty_extents(dirty_bitmap_ctx) + if is_fallback_dirty_response(extents): + allocation = nbd_backend.get_allocation_extents() + extents = [ + { + "start": e["start"], + "length": e["length"], + "dirty": True, + "zero": e["zero"], + } + for e in allocation + ] + else: + extents = backend.get_allocation_extents() + self._send_json(HTTPStatus.OK, extents) + finally: + backend.close() + except Exception as e: + logging.error("EXTENTS error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = now_s() - start + logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: + lock = self._concurrency.get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + start = now_s() + try: + logging.info("FLUSH start image_id=%s", image_id) + backend = create_backend(cfg) + try: + backend.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + finally: + backend.close() + except Exception as e: + logging.error("FLUSH error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + lock.release() + dur = now_s() - start + logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_patch_zero( + self, + image_id: str, + cfg: Dict[str, Any], + offset: int, + size: int, + flush: bool, + ) -> None: + lock = self._concurrency.get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not self._concurrency.acquire_write(image_id): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = now_s() + try: + logging.info( + "PATCH zero start image_id=%s offset=%d size=%d flush=%s", + image_id, offset, size, flush, + ) + backend = create_backend(cfg) + try: + backend.zero(offset, size) + if flush: + backend.flush() + self._send_json(HTTPStatus.OK, {"ok": True}) + except ValueError as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, str(e)) + finally: + backend.close() + except Exception as e: + logging.error("PATCH zero error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + self._concurrency.release_write(image_id) + lock.release() + dur = now_s() - start + logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) + + def _handle_patch_range( + self, + image_id: str, + cfg: Dict[str, Any], + range_header: str, + content_length: int, + ) -> None: + lock = self._concurrency.get_image_lock(image_id) + if not lock.acquire(blocking=False): + self._send_error_json(HTTPStatus.CONFLICT, "image busy") + return + + if not self._concurrency.acquire_write(image_id): + lock.release() + self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") + return + + start = now_s() + bytes_written = 0 + try: + logging.info( + "PATCH range start image_id=%s range=%s content_length=%d", + image_id, range_header, content_length, + ) + backend = create_backend(cfg) + try: + image_size = backend.size() + try: + start_off, end_inclusive = self._parse_single_range( + range_header, image_size + ) + except ValueError as e: + if "unsatisfiable" in str(e).lower(): + self._send_range_not_satisfiable(image_size) + else: + self._send_error_json( + HTTPStatus.BAD_REQUEST, f"invalid Range header: {e}" + ) + return + expected_len = end_inclusive - start_off + 1 + if content_length != expected_len: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + f"Content-Length ({content_length}) must equal range length ({expected_len})", + ) + return + nbd_backend: NbdBackend = backend # type: ignore[assignment] + bytes_written = nbd_backend.write_range(self.rfile, start_off, content_length) + self._send_json(HTTPStatus.OK, {"ok": True, "bytes_written": bytes_written}) + except ValueError: + image_size = backend.size() + self._send_range_not_satisfiable(image_size) + except IOError as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, str(e)) + finally: + backend.close() + except Exception as e: + logging.error("PATCH range error image_id=%s err=%r", image_id, e) + self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") + finally: + self._concurrency.release_write(image_id) + lock.release() + dur = now_s() - start + logging.info( + "PATCH range end image_id=%s bytes=%d duration_s=%.3f", + image_id, bytes_written, dur, + ) diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py new file mode 100644 index 000000000000..7e9cc74dcaf6 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import logging +from http.server import HTTPServer +from socketserver import ThreadingMixIn +from typing import Type + +try: + from http.server import ThreadingHTTPServer +except ImportError: + class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] + pass + +from .concurrency import ConcurrencyManager +from .config import TransferConfigLoader +from .constants import CFG_DIR, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES +from .handler import Handler + + +def make_handler( + concurrency: ConcurrencyManager, + config_loader: TransferConfigLoader, +) -> Type[Handler]: + """ + Create a Handler subclass with injected dependencies. + + BaseHTTPRequestHandler is instantiated per-request by the server, so we + cannot pass constructor args. Instead we set class-level attributes. + """ + + class ConfiguredHandler(Handler): + _concurrency = concurrency + _config_loader = config_loader + + return ConfiguredHandler + + +def main() -> None: + parser = argparse.ArgumentParser( + description="CloudStack image server backed by NBD / local file" + ) + parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") + parser.add_argument("--port", type=int, default=54323, help="Port to listen on") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + concurrency = ConcurrencyManager(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES) + config_loader = TransferConfigLoader(CFG_DIR) + handler_cls = make_handler(concurrency, config_loader) + + addr = (args.listen, args.port) + httpd = ThreadingHTTPServer(addr, handler_cls) + logging.info("listening on http://%s:%d", args.listen, args.port) + logging.info("image configs are read from %s/", config_loader.cfg_dir) + httpd.serve_forever() diff --git a/scripts/vm/hypervisor/kvm/imageserver/util.py b/scripts/vm/hypervisor/kvm/imageserver/util.py new file mode 100644 index 000000000000..71e51cec65a1 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/util.py @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import time +from typing import Any, Dict, List, Set, Tuple + + +def json_bytes(obj: Any) -> bytes: + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def merge_dirty_zero_extents( + allocation_extents: List[Tuple[int, int, bool]], + dirty_extents: List[Tuple[int, int, bool]], + size: int, +) -> List[Dict[str, Any]]: + """ + Merge allocation (start, length, zero) and dirty (start, length, dirty) extents + into a single list of {start, length, dirty, zero} with unified boundaries. + """ + boundaries: Set[int] = {0, size} + for start, length, _ in allocation_extents: + boundaries.add(start) + boundaries.add(start + length) + for start, length, _ in dirty_extents: + boundaries.add(start) + boundaries.add(start + length) + sorted_boundaries = sorted(boundaries) + + def lookup( + extents: List[Tuple[int, int, bool]], offset: int, default: bool + ) -> bool: + for start, length, flag in extents: + if start <= offset < start + length: + return flag + return default + + result: List[Dict[str, Any]] = [] + for i in range(len(sorted_boundaries) - 1): + a, b = sorted_boundaries[i], sorted_boundaries[i + 1] + if a >= b: + continue + result.append( + { + "start": a, + "length": b - a, + "dirty": lookup(dirty_extents, a, False), + "zero": lookup(allocation_extents, a, False), + } + ) + return result + + +def is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: + """True if extents is the single-extent fallback (dirty=false, zero=false).""" + return ( + len(extents) == 1 + and extents[0].get("dirty") is False + and extents[0].get("zero") is False + ) + + +def now_s() -> float: + return time.monotonic() From 81fc6d5da6bafde2752be892af57155880d5152b Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:40:04 +0530 Subject: [PATCH 071/129] Agent communication with Image server via unix socket --- .../resource/ImageServerControlSocket.java | 123 ++++++++++++++++ .../resource/LibvirtComputingResource.java | 7 +- ...virtCreateImageTransferCommandWrapper.java | 69 ++++----- ...rtFinalizeImageTransferCommandWrapper.java | 33 ++--- scripts/vm/hypervisor/kvm/image_server.py | 28 ---- .../vm/hypervisor/kvm/imageserver/__init__.py | 6 +- .../vm/hypervisor/kvm/imageserver/config.py | 128 +++++----------- .../hypervisor/kvm/imageserver/constants.py | 1 + .../vm/hypervisor/kvm/imageserver/handler.py | 8 +- .../vm/hypervisor/kvm/imageserver/server.py | 138 +++++++++++++++++- 10 files changed, 347 insertions(+), 194 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java delete mode 100644 scripts/vm/hypervisor/kvm/image_server.py diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java new file mode 100644 index 000000000000..2e9852f7bc1e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java @@ -0,0 +1,123 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Communicates with the cloudstack-image-server control socket via socat. + * + * Protocol: newline-delimited JSON over a Unix domain socket. + * Actions: register, unregister, status. + */ +public class ImageServerControlSocket { + private static final Logger LOGGER = LogManager.getLogger(ImageServerControlSocket.class); + static final String CONTROL_SOCKET_PATH = "/var/run/cloudstack/image-server.sock"; + private static final Gson GSON = new GsonBuilder().create(); + + private ImageServerControlSocket() { + } + + /** + * Send a JSON message to the image server control socket and return the + * parsed response, or null on communication failure. + */ + static JsonObject sendMessage(Map message) { + String json = GSON.toJson(message); + Script script = new Script("/bin/bash", LOGGER); + script.add("-c"); + script.add(String.format("echo '%s' | socat -t5 - UNIX-CONNECT:%s", + json.replace("'", "'\\''"), CONTROL_SOCKET_PATH)); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = script.execute(parser); + if (result != null) { + LOGGER.error("Control socket communication failed: {}", result); + return null; + } + String output = parser.getLines(); + if (output == null || output.trim().isEmpty()) { + LOGGER.error("Empty response from control socket"); + return null; + } + try { + return JsonParser.parseString(output.trim()).getAsJsonObject(); + } catch (Exception e) { + LOGGER.error("Failed to parse control socket response: {}", output, e); + return null; + } + } + + /** + * Register a transfer config with the image server. + * @return true if the server accepted the registration. + */ + public static boolean registerTransfer(String transferId, Map config) { + Map msg = new HashMap<>(); + msg.put("action", "register"); + msg.put("transfer_id", transferId); + msg.put("config", config); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } + + /** + * Unregister a transfer from the image server. + * @return the number of remaining active transfers, or -1 on error. + */ + public static int unregisterTransfer(String transferId) { + Map msg = new HashMap<>(); + msg.put("action", "unregister"); + msg.put("transfer_id", transferId); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return -1; + } + if (!"ok".equals(resp.has("status") ? resp.get("status").getAsString() : null)) { + return -1; + } + return resp.has("active_transfers") ? resp.get("active_transfers").getAsInt() : -1; + } + + /** + * Check whether the image server control socket is responsive. + * @return true if the server responded with status "ok". + */ + public static boolean isReady() { + Map msg = new HashMap<>(); + msg.put("action", "status"); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dfba9ad11157..821be05cfb21 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1100,10 +1100,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } - imageServerPath = Script.findScript(kvmScriptsDir, "image_server.py"); - if (imageServerPath == null) { - throw new ConfigurationException("Unable to find image_server.py"); + String imageServerMain = Script.findScript(kvmScriptsDir, "imageserver/__main__.py"); + if (imageServerMain == null) { + throw new ConfigurationException("Unable to find imageserver package"); } + imageServerPath = new File(imageServerMain).getParent(); createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index db0918f5c072..71beafe9fa1b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -18,7 +18,6 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.File; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -26,62 +25,60 @@ import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.storage.resource.IpTablesHelper; -import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; -import com.google.gson.GsonBuilder; @ResourceWrapper(handles = CreateImageTransferCommand.class) public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputingResource resource) { - final String imageServerScript = resource.getImageServerPath(); + final String imageServerPackageDir = resource.getImageServerPath(); + final String imageServerParentDir = new File(imageServerPackageDir).getParent(); + final String imageServerModuleName = new File(imageServerPackageDir).getName(); String unitName = "cloudstack-image-server"; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); String checkResult = checkScript.execute(); - if (checkResult == null) { + if (checkResult == null && ImageServerControlSocket.isReady()) { return true; } - String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no /usr/bin/python3 %s --listen 0.0.0.0 --port %d", - unitName, imageServerScript, imageServerPort); + if (checkResult != null) { + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen 0.0.0.0 --port %d", + unitName, imageServerParentDir, imageServerModuleName, imageServerPort); - Script startScript = new Script("/bin/bash", logger); - startScript.add("-c"); - startScript.add(systemdRunCmd); - String startResult = startScript.execute(); + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); - if (startResult != null) { - logger.error(String.format("Failed to start the Image server: %s", startResult)); - return false; + if (startResult != null) { + logger.error(String.format("Failed to start the Image server: %s", startResult)); + return false; + } } - // Wait with timeout until the service is up int maxWaitSeconds = 10; int pollIntervalMs = 1000; int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; - boolean serviceActive = false; + boolean serverReady = false; for (int attempt = 0; attempt < maxAttempts; attempt++) { - Script verifyScript = new Script("/bin/bash", logger); - verifyScript.add("-c"); - verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); - String verifyResult = verifyScript.execute(); - if (verifyResult == null) { - serviceActive = true; - logger.info(String.format("Image server is now active (attempt %d)", attempt + 1)); + if (ImageServerControlSocket.isReady()) { + serverReady = true; + logger.info(String.format("Image server control socket is ready (attempt %d)", attempt + 1)); break; } try { @@ -92,8 +89,8 @@ private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputi } } - if (!serviceActive) { - logger.error(String.format("Image server failed to start within %d seconds", maxWaitSeconds)); + if (!serverReady) { + logger.error(String.format("Image server control socket not ready within %d seconds", maxWaitSeconds)); return false; } @@ -138,22 +135,14 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r } } - try { - final String json = new GsonBuilder().create().toJson(payload); - File dir = new File("/tmp/imagetransfer"); - if (!dir.exists()) { - dir.mkdirs(); - } - final File transferFile = new File("/tmp/imagetransfer", transferId); - FileUtils.writeStringToFile(transferFile, json, "UTF-8"); - - } catch (IOException e) { - logger.warn("Failed to prepare image transfer on KVM host", e); - return new CreateImageTransferAnswer(cmd, false, "Failed to prepare image transfer on KVM host: " + e.getMessage()); + final int imageServerPort = 54323; + if (!startImageServerIfNotRunning(imageServerPort, resource)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); } - final int imageServerPort = 54323; - startImageServerIfNotRunning(imageServerPort, resource); + if (!ImageServerControlSocket.registerTransfer(transferId, payload)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to register transfer with image server."); + } final String transferUrl = String.format("http://%s:%d/images/%s", resource.getPrivateIp(), imageServerPort, transferId); return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java index c2c9d7a797d6..3f0026ae38f2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -17,18 +17,12 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; - import org.apache.cloudstack.backup.FinalizeImageTransferCommand; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; @@ -54,9 +48,8 @@ private boolean stopImageServer() { checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); String checkResult = checkScript.execute(); if (checkResult != null) { - logger.info(String.format("Image server not running, resetting failed state")); + logger.info("Image server not running, resetting failed state"); resetService(unitName); - // Still try to remove firewall rule in case it exists removeFirewallRule(imageServerPort); return true; } @@ -66,7 +59,7 @@ private boolean stopImageServer() { stopScript.add(String.format("systemctl stop %s", unitName)); stopScript.execute(); resetService(unitName); - logger.info(String.format("Image server %s stopped", unitName)); + logger.info("Image server {} stopped", unitName); removeFirewallRule(imageServerPort); @@ -80,9 +73,9 @@ private void removeFirewallRule(int port) { removeScript.add(String.format("iptables -D INPUT %s || true", rule)); String result = removeScript.execute(); if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { - logger.debug(String.format("Firewall rule removal result for port %d: %s", port, result)); + logger.debug("Firewall rule removal result for port {}: {}", port, result); } else { - logger.info(String.format("Firewall rule removed for port %d (or did not exist)", port)); + logger.info("Firewall rule removed for port {} (or did not exist)", port); } } @@ -92,17 +85,15 @@ public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource return new Answer(cmd, false, "transferId is empty."); } - final File transferFile = new File("/tmp/imagetransfer", transferId); - if (transferFile.exists() && !transferFile.delete()) { - return new Answer(cmd, false, "Failed to delete transfer config file: " + transferFile.getAbsolutePath()); + int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); + if (activeTransfers < 0) { + logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); + stopImageServer(); + return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); } - try (Stream stream = Files.list(Paths.get("/tmp/imagetransfer"))) { - if (!stream.findAny().isPresent()) { - stopImageServer(); - } - } catch (IOException e) { - logger.warn("Failed to list /tmp/imagetransfer", e); + if (activeTransfers == 0) { + stopImageServer(); } return new Answer(cmd, true, "Image transfer finalized."); diff --git a/scripts/vm/hypervisor/kvm/image_server.py b/scripts/vm/hypervisor/kvm/image_server.py deleted file mode 100644 index c0436b4d2077..000000000000 --- a/scripts/vm/hypervisor/kvm/image_server.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from imageserver.server import main - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/vm/hypervisor/kvm/imageserver/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/__init__.py index 5e033f5d527f..69eec98956a3 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/__init__.py +++ b/scripts/vm/hypervisor/kvm/imageserver/__init__.py @@ -18,7 +18,11 @@ """ CloudStack image server — HTTP server backed by NBD over Unix socket or a local file. -Supports two backends (configured per-transfer via JSON config): +Transfer configs are registered/unregistered by the cloudstack-agent via a +Unix domain control socket (default: /var/run/cloudstack/image-server.sock) +and stored in-memory for the lifetime of the server process. + +Supports two backends (configured per-transfer at registration time): - nbd: proxy to an NBD server via Unix socket; supports range reads/writes (GET/PUT/PATCH), extents, zero, flush. - file: read/write a local qcow2/raw file; full PUT only, GET with optional diff --git a/scripts/vm/hypervisor/kvm/imageserver/config.py b/scripts/vm/hypervisor/kvm/imageserver/config.py index cc0107cce9d0..3b1fd686f053 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/config.py +++ b/scripts/vm/hypervisor/kvm/imageserver/config.py @@ -15,13 +15,10 @@ # specific language governing permissions and limitations # under the License. -import json import logging import os import threading -from typing import Any, Dict, Optional, Tuple - -from .constants import CFG_DIR +from typing import Any, Dict, Optional def safe_transfer_id(image_id: str) -> Optional[str]: @@ -40,97 +37,48 @@ def safe_transfer_id(image_id: str) -> Optional[str]: return image_id -class TransferConfigLoader: +class TransferRegistry: """ - Loads and caches per-image transfer configuration from JSON files. + Thread-safe in-memory registry for active image transfer configurations. - CloudStack writes a JSON file at / with: - - NBD backend: {"backend": "nbd", "socket": "...", "export": "vda", "export_bitmap": "..."} - - File backend: {"backend": "file", "file": "/path/to/image.qcow2"} + The cloudstack-agent registers/unregisters transfers via the Unix domain + control socket. The HTTP handler looks up configs through get(). """ - def __init__(self, cfg_dir: str = CFG_DIR): - self._cfg_dir = cfg_dir - self._cache: Dict[str, Tuple[float, Dict[str, Any]]] = {} - self._cache_guard = threading.Lock() - - @property - def cfg_dir(self) -> str: - return self._cfg_dir + def __init__(self) -> None: + self._lock = threading.Lock() + self._transfers: Dict[str, Dict[str, Any]] = {} - def load(self, image_id: str) -> Optional[Dict[str, Any]]: - safe_id = safe_transfer_id(image_id) + def register(self, transfer_id: str, config: Dict[str, Any]) -> bool: + safe_id = safe_transfer_id(transfer_id) + if safe_id is None: + logging.error("register rejected invalid transfer_id=%r", transfer_id) + return False + with self._lock: + self._transfers[safe_id] = config + logging.info("registered transfer_id=%s active=%d", safe_id, len(self._transfers)) + return True + + def unregister(self, transfer_id: str) -> int: + """Remove a transfer and return the number of remaining active transfers.""" + safe_id = safe_transfer_id(transfer_id) + if safe_id is None: + logging.error("unregister rejected invalid transfer_id=%r", transfer_id) + with self._lock: + return len(self._transfers) + with self._lock: + self._transfers.pop(safe_id, None) + remaining = len(self._transfers) + logging.info("unregistered transfer_id=%s active=%d", safe_id, remaining) + return remaining + + def get(self, transfer_id: str) -> Optional[Dict[str, Any]]: + safe_id = safe_transfer_id(transfer_id) if safe_id is None: return None + with self._lock: + return self._transfers.get(safe_id) - cfg_path = os.path.join(self._cfg_dir, safe_id) - try: - st = os.stat(cfg_path) - except FileNotFoundError: - return None - except OSError as e: - logging.error("cfg stat failed image_id=%s err=%r", image_id, e) - return None - - with self._cache_guard: - cached = self._cache.get(safe_id) - if cached is not None: - cached_mtime, cached_cfg = cached - if float(st.st_mtime) == float(cached_mtime): - return cached_cfg - - try: - with open(cfg_path, "rb") as f: - raw = f.read(4096) - except OSError as e: - logging.error("cfg read failed image_id=%s err=%r", image_id, e) - return None - - try: - obj = json.loads(raw.decode("utf-8")) - except Exception as e: - logging.error("cfg parse failed image_id=%s err=%r", image_id, e) - return None - - if not isinstance(obj, dict): - logging.error("cfg invalid type image_id=%s type=%s", image_id, type(obj).__name__) - return None - - backend = obj.get("backend") - if backend is None: - backend = "nbd" - if not isinstance(backend, str): - logging.error("cfg invalid backend type image_id=%s", image_id) - return None - backend = backend.lower() - if backend not in ("nbd", "file"): - logging.error("cfg unsupported backend image_id=%s backend=%s", image_id, backend) - return None - - if backend == "file": - file_path = obj.get("file") - if not isinstance(file_path, str) or not file_path.strip(): - logging.error("cfg missing/invalid file path for file backend image_id=%s", image_id) - return None - cfg: Dict[str, Any] = {"backend": "file", "file": file_path.strip()} - else: - socket_path = obj.get("socket") - export = obj.get("export") - export_bitmap = obj.get("export_bitmap") - if not isinstance(socket_path, str) or not socket_path.strip(): - logging.error("cfg missing/invalid socket path for nbd backend image_id=%s", image_id) - return None - socket_path = socket_path.strip() - if export is not None and (not isinstance(export, str) or not export): - logging.error("cfg missing/invalid export image_id=%s", image_id) - return None - cfg = { - "backend": "nbd", - "socket": socket_path, - "export": export, - "export_bitmap": export_bitmap, - } - - with self._cache_guard: - self._cache[safe_id] = (float(st.st_mtime), cfg) - return cfg + def active_count(self) -> int: + with self._lock: + return len(self._transfers) diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py index 6836f5798070..4e8d5c86da5b 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/constants.py +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -27,3 +27,4 @@ MAX_PARALLEL_WRITES = 1 CFG_DIR = "/tmp/imagetransfer" +CONTROL_SOCKET = "/var/run/cloudstack/image-server.sock" diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index 8d894f9b0c58..a689467238bd 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -25,7 +25,7 @@ from .backends import NbdBackend, create_backend from .concurrency import ConcurrencyManager -from .config import TransferConfigLoader +from .config import TransferRegistry from .constants import CHUNK_SIZE, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES from .util import is_fallback_dirty_response, json_bytes, now_s @@ -38,7 +38,7 @@ class Handler(BaseHTTPRequestHandler): All backend I/O is delegated to ImageBackend implementations via the create_backend() factory. - Class-level attributes _concurrency and _config_loader are injected + Class-level attributes _concurrency and _registry are injected by the server at startup (see server.py / make_handler()). """ @@ -46,7 +46,7 @@ class Handler(BaseHTTPRequestHandler): server_protocol = "HTTP/1.1" _concurrency: ConcurrencyManager - _config_loader: TransferConfigLoader + _registry: TransferRegistry _CONTENT_RANGE_RE = re.compile(r"^bytes\s+(\d+)-(\d+)/(?:\*|\d+)$") @@ -197,7 +197,7 @@ def _parse_query(self) -> Dict[str, List[str]]: return parse_qs(query, keep_blank_values=True) def _image_cfg(self, image_id: str) -> Optional[Dict[str, Any]]: - return self._config_loader.load(image_id) + return self._registry.get(image_id) # ------------------------------------------------------------------ # HTTP verb dispatchers diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index 7e9cc74dcaf6..d348bf4950db 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -16,7 +16,11 @@ # under the License. import argparse +import json import logging +import os +import socket +import threading from http.server import HTTPServer from socketserver import ThreadingMixIn from typing import Type @@ -28,14 +32,14 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] pass from .concurrency import ConcurrencyManager -from .config import TransferConfigLoader -from .constants import CFG_DIR, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES +from .config import TransferRegistry +from .constants import CONTROL_SOCKET, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES from .handler import Handler def make_handler( concurrency: ConcurrencyManager, - config_loader: TransferConfigLoader, + registry: TransferRegistry, ) -> Type[Handler]: """ Create a Handler subclass with injected dependencies. @@ -46,17 +50,131 @@ def make_handler( class ConfiguredHandler(Handler): _concurrency = concurrency - _config_loader = config_loader + _registry = registry return ConfiguredHandler +def _validate_config(obj: dict) -> dict: + """ + Validate and normalize a transfer config dict received over the control + socket. Returns the cleaned config or raises ValueError. + """ + backend = obj.get("backend") + if backend is None: + backend = "nbd" + if not isinstance(backend, str): + raise ValueError("invalid backend type") + backend = backend.lower() + if backend not in ("nbd", "file"): + raise ValueError(f"unsupported backend: {backend}") + + if backend == "file": + file_path = obj.get("file") + if not isinstance(file_path, str) or not file_path.strip(): + raise ValueError("missing/invalid file path for file backend") + return {"backend": "file", "file": file_path.strip()} + + socket_path = obj.get("socket") + export = obj.get("export") + export_bitmap = obj.get("export_bitmap") + if not isinstance(socket_path, str) or not socket_path.strip(): + raise ValueError("missing/invalid socket path for nbd backend") + if export is not None and (not isinstance(export, str) or not export): + raise ValueError("invalid export name") + return { + "backend": "nbd", + "socket": socket_path.strip(), + "export": export, + "export_bitmap": export_bitmap, + } + + +def _handle_control_conn(conn: socket.socket, registry: TransferRegistry) -> None: + """Handle a single control-socket connection (one JSON request/response).""" + try: + data = b"" + while True: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + if b"\n" in data: + break + + msg = json.loads(data.strip()) + action = msg.get("action") + + if action == "register": + transfer_id = msg.get("transfer_id") + raw_config = msg.get("config") + if not transfer_id or not isinstance(raw_config, dict): + resp = {"status": "error", "message": "missing transfer_id or config"} + else: + try: + config = _validate_config(raw_config) + except ValueError as e: + resp = {"status": "error", "message": str(e)} + else: + if registry.register(transfer_id, config): + resp = {"status": "ok", "active_transfers": registry.active_count()} + else: + resp = {"status": "error", "message": "invalid transfer_id"} + elif action == "unregister": + transfer_id = msg.get("transfer_id") + if not transfer_id: + resp = {"status": "error", "message": "missing transfer_id"} + else: + remaining = registry.unregister(transfer_id) + resp = {"status": "ok", "active_transfers": remaining} + elif action == "status": + resp = {"status": "ok", "active_transfers": registry.active_count()} + else: + resp = {"status": "error", "message": f"unknown action: {action}"} + + conn.sendall((json.dumps(resp) + "\n").encode("utf-8")) + except Exception as e: + logging.error("control socket error: %r", e) + try: + conn.sendall((json.dumps({"status": "error", "message": str(e)}) + "\n").encode("utf-8")) + except Exception: + pass + finally: + conn.close() + + +def _control_listener(registry: TransferRegistry, sock_path: str) -> None: + """Accept loop for the Unix domain control socket (runs in a daemon thread).""" + if os.path.exists(sock_path): + os.unlink(sock_path) + os.makedirs(os.path.dirname(sock_path), exist_ok=True) + + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + srv.bind(sock_path) + os.chmod(sock_path, 0o660) + srv.listen(5) + logging.info("control socket listening on %s", sock_path) + + while True: + conn, _ = srv.accept() + threading.Thread( + target=_handle_control_conn, + args=(conn, registry), + daemon=True, + ).start() + + def main() -> None: parser = argparse.ArgumentParser( description="CloudStack image server backed by NBD / local file" ) parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") parser.add_argument("--port", type=int, default=54323, help="Port to listen on") + parser.add_argument( + "--control-socket", + default=CONTROL_SOCKET, + help="Path to the Unix domain control socket", + ) args = parser.parse_args() logging.basicConfig( @@ -64,12 +182,18 @@ def main() -> None: format="%(asctime)s %(levelname)s %(message)s", ) + registry = TransferRegistry() concurrency = ConcurrencyManager(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES) - config_loader = TransferConfigLoader(CFG_DIR) - handler_cls = make_handler(concurrency, config_loader) + handler_cls = make_handler(concurrency, registry) + + ctrl_thread = threading.Thread( + target=_control_listener, + args=(registry, args.control_socket), + daemon=True, + ) + ctrl_thread.start() addr = (args.listen, args.port) httpd = ThreadingHTTPServer(addr, handler_cls) logging.info("listening on http://%s:%d", args.listen, args.port) - logging.info("image configs are read from %s/", config_loader.cfg_dir) httpd.serve_forever() From dad314a8a6d804621f02b65c358b2e4dca88b1b1 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 23 Mar 2026 20:53:37 +0530 Subject: [PATCH 072/129] Image server unittests --- .../kvm/imageserver/tests/__init__.py | 16 + .../kvm/imageserver/tests/test_base.py | 440 ++++++++++++++++++ .../imageserver/tests/test_combinations.py | 397 ++++++++++++++++ .../imageserver/tests/test_control_socket.py | 258 ++++++++++ .../imageserver/tests/test_file_backend.py | 230 +++++++++ .../kvm/imageserver/tests/test_nbd_backend.py | 393 ++++++++++++++++ 6 files changed, 1734 insertions(+) create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_combinations.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_control_socket.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_file_backend.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py new file mode 100644 index 000000000000..0ccbeeeafb7c --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py new file mode 100644 index 000000000000..91e7eda79ed4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py @@ -0,0 +1,440 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared infrastructure for the image-server test suite (stdlib unittest only). + +Provides: +- A singleton image server process started once for the entire test run. +- Control-socket helpers using pure-Python AF_UNIX (no socat). +- qemu-nbd server management. +- Transfer registration / teardown helpers. +- HTTP helper functions. +""" + +import functools +import json +import logging +import os +import random +import select +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import unittest +import uuid +from pathlib import Path +from typing import Any, Dict, Optional + +IMAGE_SIZE = 1 * 1024 * 1024 # 1 MiB +SERVER_STARTUP_TIMEOUT = 10 +QEMU_NBD_STARTUP_TIMEOUT = 5 +HTTP_TIMEOUT = 30 # seconds per HTTP request + +logging.basicConfig( + level=logging.INFO, + stream=sys.stderr, + format="%(asctime)s [TEST] %(message)s", +) +log = logging.getLogger(__name__) + + +def randbytes(seed, n): + """Generate n deterministic pseudo-random bytes (works on Python 3.6+).""" + rng = random.Random(seed) + return rng.getrandbits(8 * n).to_bytes(n, "big") + + +def test_timeout(seconds): + """Decorator that fails a test if it exceeds *seconds* (SIGALRM, Unix only).""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + def _alarm(signum, frame): + raise TimeoutError( + "{} timed out after {}s".format(func.__qualname__, seconds) + ) + prev = signal.signal(signal.SIGALRM, _alarm) + signal.alarm(seconds) + try: + return func(*args, **kwargs) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev) + return wrapper + return decorator + +# ── Singleton state shared across all test modules ────────────────────── + +_tmp_dir: Optional[str] = None +_server_proc: Optional[subprocess.Popen] = None +_server_info: Optional[Dict[str, Any]] = None + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def control_socket_send(sock_path: str, message: dict, retries: int = 5) -> dict: + """Send a JSON message to the control socket and return the parsed response.""" + payload = (json.dumps(message) + "\n").encode("utf-8") + last_err = None + for attempt in range(retries): + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(5) + s.connect(sock_path) + s.sendall(payload) + s.shutdown(socket.SHUT_WR) + data = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + return json.loads(data.strip()) + except (BlockingIOError, ConnectionRefusedError, OSError) as e: + last_err = e + time.sleep(0.1 * (attempt + 1)) + raise last_err + + +def _wait_for_control_socket(sock_path: str, timeout: float = SERVER_STARTUP_TIMEOUT) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + resp = control_socket_send(sock_path, {"action": "status"}) + if resp.get("status") == "ok": + return + except (ConnectionRefusedError, FileNotFoundError, OSError): + pass + time.sleep(0.2) + raise RuntimeError( + f"Image server control socket at {sock_path} not ready within {timeout}s" + ) + + +def _wait_for_nbd_socket(sock_path: str, timeout: float = QEMU_NBD_STARTUP_TIMEOUT) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if os.path.exists(sock_path): + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(1) + s.connect(sock_path) + return + except (ConnectionRefusedError, OSError): + pass + time.sleep(0.2) + raise RuntimeError( + f"qemu-nbd socket at {sock_path} not ready within {timeout}s" + ) + + +def get_tmp_dir() -> str: + global _tmp_dir + if _tmp_dir is None: + _tmp_dir = tempfile.mkdtemp(prefix="imageserver_test_") + return _tmp_dir + + +def get_image_server() -> Dict[str, Any]: + """Return the singleton image-server info dict, starting it if needed.""" + global _server_proc, _server_info + + if _server_info is not None: + return _server_info + + tmp = get_tmp_dir() + port = _free_port() + ctrl_sock = os.path.join(tmp, "ctrl.sock") + + imageserver_pkg = str(Path(__file__).resolve().parent.parent) + parent_dir = str(Path(imageserver_pkg).parent) + + env = os.environ.copy() + env["PYTHONPATH"] = parent_dir + os.pathsep + env.get("PYTHONPATH", "") + + proc = subprocess.Popen( + [ + sys.executable, "-m", "imageserver", + "--listen", "127.0.0.1", + "--port", str(port), + "--control-socket", ctrl_sock, + ], + cwd=parent_dir, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _server_proc = proc + + try: + _wait_for_control_socket(ctrl_sock) + except RuntimeError: + proc.kill() + stdout, stderr = proc.communicate(timeout=5) + raise RuntimeError( + f"Image server failed to start.\nstdout: {stdout.decode()}\nstderr: {stderr.decode()}" + ) + + def send(msg: dict) -> dict: + return control_socket_send(ctrl_sock, msg) + + _server_info = { + "base_url": f"http://127.0.0.1:{port}", + "port": port, + "ctrl_sock": ctrl_sock, + "send": send, + } + return _server_info + + +def shutdown_image_server() -> None: + global _server_proc, _server_info, _tmp_dir + if _server_proc is not None: + for pipe in (_server_proc.stdout, _server_proc.stderr): + if pipe: + try: + pipe.close() + except Exception: + pass + _server_proc.terminate() + try: + _server_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + _server_proc.kill() + _server_proc.wait(timeout=5) + _server_proc = None + _server_info = None + if _tmp_dir is not None: + shutil.rmtree(_tmp_dir, ignore_errors=True) + _tmp_dir = None + + +# ── qemu-nbd server ──────────────────────────────────────────────────── + +class QemuNbdServer: + """Manages a qemu-nbd process exporting a raw image over a Unix socket.""" + + def __init__(self, image_path: str, socket_path: str, image_size: int = IMAGE_SIZE): + self.image_path = image_path + self.socket_path = socket_path + self.image_size = image_size + self._proc: Optional[subprocess.Popen] = None + + def start(self) -> None: + if not os.path.exists(self.image_path): + with open(self.image_path, "wb") as f: + f.truncate(self.image_size) + + self._proc = subprocess.Popen( + [ + "qemu-nbd", + "--socket", self.socket_path, + "--format", "raw", + "--persistent", + "--shared=8", + "--cache=none", + self.image_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _wait_for_nbd_socket(self.socket_path) + + def stop(self) -> None: + if self._proc is not None: + for pipe in (self._proc.stdout, self._proc.stderr): + if pipe: + try: + pipe.close() + except Exception: + pass + self._proc.terminate() + try: + self._proc.wait(timeout=5) + except subprocess.TimeoutExpired: + self._proc.kill() + self._proc.wait(timeout=5) + self._proc = None + + +# ── Factory helpers ───────────────────────────────────────────────────── + +def make_tmp_image(data=None, image_size=IMAGE_SIZE) -> str: + """Create a temp raw image file in the shared tmp dir; return path.""" + tmp = get_tmp_dir() + path = os.path.join(tmp, f"img_{uuid.uuid4().hex[:8]}.raw") + if data is not None: + with open(path, "wb") as f: + f.write(data) + else: + with open(path, "wb") as f: + f.write(randbytes(42, image_size)) + return path + + +def make_file_transfer(data=None, image_size=IMAGE_SIZE): + """ + Create a temp file + register a file-backend transfer. + Returns (transfer_id, url, file_path, cleanup_callable). + """ + srv = get_image_server() + path = make_tmp_image(data=data, image_size=image_size) + transfer_id = f"file-{uuid.uuid4().hex[:8]}" + resp = srv["send"]({ + "action": "register", + "transfer_id": transfer_id, + "config": {"backend": "file", "file": path}, + }) + assert resp["status"] == "ok", f"register failed: {resp}" + url = f"{srv['base_url']}/images/{transfer_id}" + + def cleanup(): + srv["send"]({"action": "unregister", "transfer_id": transfer_id}) + try: + os.unlink(path) + except FileNotFoundError: + pass + + return transfer_id, url, path, cleanup + + +def make_nbd_transfer(image_size=IMAGE_SIZE): + """ + Create a qemu-nbd server + register an NBD-backend transfer. + Returns (transfer_id, url, QemuNbdServer, cleanup_callable). + """ + srv = get_image_server() + tmp = get_tmp_dir() + img_path = os.path.join(tmp, f"nbd_{uuid.uuid4().hex[:8]}.raw") + sock_path = os.path.join(tmp, f"nbd_{uuid.uuid4().hex[:8]}.sock") + + server = QemuNbdServer(img_path, sock_path, image_size=image_size) + server.start() + + transfer_id = f"nbd-{uuid.uuid4().hex[:8]}" + resp = srv["send"]({ + "action": "register", + "transfer_id": transfer_id, + "config": {"backend": "nbd", "socket": sock_path}, + }) + assert resp["status"] == "ok", f"register failed: {resp}" + url = f"{srv['base_url']}/images/{transfer_id}" + + def cleanup(): + srv["send"]({"action": "unregister", "transfer_id": transfer_id}) + server.stop() + for p in (img_path, sock_path): + try: + os.unlink(p) + except FileNotFoundError: + pass + + return transfer_id, url, server, cleanup + + +# ── HTTP helpers ──────────────────────────────────────────────────────── + +import urllib.request +import urllib.error + + +def http_get(url, headers=None, timeout=HTTP_TIMEOUT): + req = urllib.request.Request(url, headers=headers or {}) + return urllib.request.urlopen(req, timeout=timeout) + + +def http_put(url, data, headers=None, timeout=HTTP_TIMEOUT): + hdrs = {"Content-Length": str(len(data))} + if headers: + hdrs.update(headers) + req = urllib.request.Request(url, data=data, headers=hdrs, method="PUT") + return urllib.request.urlopen(req, timeout=timeout) + + +def http_post(url, data=b"", headers=None, timeout=HTTP_TIMEOUT): + hdrs = {} + if headers: + hdrs.update(headers) + req = urllib.request.Request(url, data=data, headers=hdrs, method="POST") + return urllib.request.urlopen(req, timeout=timeout) + + +def http_options(url, timeout=HTTP_TIMEOUT): + req = urllib.request.Request(url, method="OPTIONS") + return urllib.request.urlopen(req, timeout=timeout) + + +def http_patch(url, data, headers=None, timeout=HTTP_TIMEOUT): + hdrs = {} + if headers: + hdrs.update(headers) + req = urllib.request.Request(url, data=data, headers=hdrs, method="PATCH") + return urllib.request.urlopen(req, timeout=timeout) + + +# ── Base TestCase with shared setUp/tearDown ──────────────────────────── + +class ImageServerTestCase(unittest.TestCase): + """ + Base class for image-server tests. + + Ensures the image server is running before any test method. + Subclasses that need a file or NBD transfer should set them up + in setUp() and tear down in tearDown(). + """ + + @classmethod + def setUpClass(cls): + cls.server = get_image_server() + cls.base_url = cls.server["base_url"] + + def ctrl(self, msg): + """Send a control-socket message; wraps server['send'] to avoid descriptor issues.""" + return self.server["send"](msg) + + def _make_tmp_image(self, data=None): + return make_tmp_image(data=data) + + def _register_file_transfer(self, data=None): + return make_file_transfer(data=data) + + def _register_nbd_transfer(self): + return make_nbd_transfer() + + @staticmethod + def dump_server_logs(): + """Read any available server stderr and print it for post-mortem debugging.""" + if _server_proc is None or _server_proc.stderr is None: + return + try: + if select.select([_server_proc.stderr], [], [], 0)[0]: + data = _server_proc.stderr.read1(64 * 1024) + if data: + sys.stderr.write("\n=== IMAGE SERVER STDERR ===\n") + sys.stderr.write(data.decode(errors="replace")) + sys.stderr.write("\n=== END SERVER STDERR ===\n") + except Exception: + pass diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_combinations.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_combinations.py new file mode 100644 index 000000000000..509f9fde05a5 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_combinations.py @@ -0,0 +1,397 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Multi-operation sequences, parallel reads across multiple transfer objects, +cross-backend scenarios, and edge cases. +""" + +import json +import logging +import unittest +import urllib.error +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .test_base import ( + IMAGE_SIZE, + ImageServerTestCase, + http_get, + http_patch, + http_post, + http_put, + make_file_transfer, + make_nbd_transfer, + randbytes, + shutdown_image_server, + test_timeout, +) + +log = logging.getLogger(__name__) +FUTURES_TIMEOUT = 60 # seconds for as_completed to collect all results + + +def _fetch(url, headers=None): + """GET *url* and return the body bytes, properly closing the response.""" + resp = http_get(url, headers=headers) + try: + return resp.read() + finally: + resp.close() + + +class TestParallelReadsFileBackend(ImageServerTestCase): + """Multiple concurrent GET requests to multiple file-backed transfers.""" + + @test_timeout(120) + def test_parallel_reads_single_file_transfer(self): + data = randbytes(500, IMAGE_SIZE) + tid, url, path, cleanup = make_file_transfer(data=data) + try: + results = {} + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for i in range(8): + start = i * (IMAGE_SIZE // 8) + end = start + (IMAGE_SIZE // 8) - 1 + f = pool.submit( + _fetch, url, headers={"Range": f"bytes={start}-{end}"} + ) + futures[f] = (start, end) + + for f in as_completed(futures, timeout=FUTURES_TIMEOUT): + start, end = futures[f] + results[(start, end)] = f.result() + + for (start, end), chunk in sorted(results.items()): + self.assertEqual(chunk, data[start:end + 1], f"Mismatch at {start}-{end}") + finally: + cleanup() + + @test_timeout(120) + def test_parallel_reads_multiple_file_transfers(self): + """Parallel reads across 4 different file-backed transfer objects.""" + transfers = [] + try: + for i in range(4): + data = randbytes(600 + i, IMAGE_SIZE) + tid, url, path, cleanup = make_file_transfer(data=data) + transfers.append((tid, url, data, cleanup)) + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for idx, (tid, url, data, _) in enumerate(transfers): + for j in range(2): + f = pool.submit(_fetch, url) + futures[f] = (idx, data) + + for f in as_completed(futures, timeout=FUTURES_TIMEOUT): + idx, expected_data = futures[f] + got = f.result() + self.assertEqual(got, expected_data, f"Transfer {idx} mismatch") + finally: + for _, _, _, cleanup in transfers: + cleanup() + + +class TestParallelReadsNbdBackend(ImageServerTestCase): + """Multiple concurrent GET requests to multiple NBD-backed transfers.""" + + @test_timeout(120) + def test_parallel_reads_single_nbd_transfer(self): + data = randbytes(700, IMAGE_SIZE) + tid, url, nbd_server, cleanup = make_nbd_transfer() + try: + log.info("Writing %d bytes to NBD transfer %s", IMAGE_SIZE, tid) + http_put(url, data) + log.info("NBD write done, starting 8 parallel range reads") + + results = {} + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for i in range(8): + start = i * (IMAGE_SIZE // 8) + end = start + (IMAGE_SIZE // 8) - 1 + f = pool.submit( + _fetch, url, headers={"Range": f"bytes={start}-{end}"} + ) + futures[f] = (start, end) + + completed = 0 + for f in as_completed(futures, timeout=FUTURES_TIMEOUT): + start, end = futures[f] + results[(start, end)] = f.result() + completed += 1 + log.info("NBD range read %d/8 done: bytes=%d-%d", completed, start, end) + + for (start, end), chunk in sorted(results.items()): + self.assertEqual(chunk, data[start:end + 1], f"Mismatch at {start}-{end}") + finally: + cleanup() + + @test_timeout(120) + def test_parallel_reads_multiple_nbd_transfers(self): + """Parallel reads across 4 different NBD-backed transfer objects.""" + transfers = [] + try: + for i in range(4): + data = randbytes(800 + i, IMAGE_SIZE) + log.info("Setting up NBD transfer %d", i) + tid, url, nbd_server, cleanup = make_nbd_transfer() + log.info("Writing data to NBD transfer %d (tid=%s)", i, tid) + http_put(url, data) + transfers.append((tid, url, data, cleanup)) + log.info("NBD transfer %d ready", i) + + log.info("Starting parallel reads across %d NBD transfers", len(transfers)) + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for idx, (tid, url, data, _) in enumerate(transfers): + for j in range(2): + f = pool.submit(_fetch, url) + futures[f] = (idx, data) + + completed = 0 + for f in as_completed(futures, timeout=FUTURES_TIMEOUT): + idx, expected_data = futures[f] + got = f.result() + completed += 1 + log.info("Read %d/%d done: NBD transfer idx=%d, %d bytes", + completed, len(futures), idx, len(got)) + self.assertEqual(got, expected_data, f"NBD transfer {idx} mismatch") + finally: + for _, _, _, cleanup in transfers: + cleanup() + + +class TestParallelReadsMixedBackends(ImageServerTestCase): + """Parallel reads across a mix of file and NBD transfers simultaneously.""" + + @test_timeout(120) + def test_parallel_reads_file_and_nbd_mixed(self): + transfers = [] + try: + for i in range(2): + log.info("Setting up file transfer %d", i) + data = randbytes(900 + i, IMAGE_SIZE) + tid, url, path, cleanup = make_file_transfer(data=data) + transfers.append(("file", tid, url, data, cleanup)) + log.info("File transfer %d ready: tid=%s", i, tid) + + for i in range(2): + log.info("Setting up NBD transfer %d", i) + data = randbytes(950 + i, IMAGE_SIZE) + tid, url, nbd_server, cleanup = make_nbd_transfer() + log.info("NBD transfer %d registered: tid=%s, writing data...", i, tid) + http_put(url, data) + transfers.append(("nbd", tid, url, data, cleanup)) + log.info("NBD transfer %d ready", i) + + log.info("Starting parallel reads across %d transfers (2 file + 2 nbd)", + len(transfers)) + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for idx, (backend_type, tid, url, data, _) in enumerate(transfers): + for j in range(2): + f = pool.submit(_fetch, url) + futures[f] = (idx, backend_type, data) + + completed = 0 + for f in as_completed(futures, timeout=FUTURES_TIMEOUT): + idx, backend_type, expected = futures[f] + got = f.result() + completed += 1 + log.info("Read %d/%d done: %s transfer idx=%d, %d bytes", + completed, len(futures), backend_type, idx, len(got)) + self.assertEqual(got, expected, f"{backend_type} transfer {idx} mismatch") + + log.info("All parallel mixed reads completed successfully") + except TimeoutError: + log.error("TIMEOUT in mixed parallel reads — dumping server logs") + self.dump_server_logs() + raise + finally: + for _, _, _, _, cleanup in transfers: + cleanup() + + +class TestWriteThenReadNbd(ImageServerTestCase): + """Multi-step write sequences on NBD backend.""" + + def setUp(self): + self._tid, self._url, self._nbd, self._cleanup = make_nbd_transfer() + + def tearDown(self): + self._cleanup() + + def test_partial_writes_then_full_read(self): + http_put(self._url, b"\x00" * IMAGE_SIZE) + + chunk_size = IMAGE_SIZE // 4 + for i in range(4): + offset = i * chunk_size + end = offset + chunk_size - 1 + data = bytes([i & 0xFF]) * chunk_size + http_patch(self._url, data, headers={ + "Range": f"bytes={offset}-{end}", + "Content-Type": "application/octet-stream", + "Content-Length": str(chunk_size), + }) + + resp = http_get(self._url) + full = resp.read() + for i in range(4): + offset = i * chunk_size + self.assertEqual(full[offset:offset + chunk_size], bytes([i & 0xFF]) * chunk_size) + + def test_zero_then_extents(self): + http_put(self._url, randbytes(1000, IMAGE_SIZE)) + + payload = json.dumps({"op": "zero", "size": IMAGE_SIZE // 2, "offset": 0}).encode() + http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + + resp = http_get(f"{self._url}/extents") + extents = json.loads(resp.read()) + total = sum(e["length"] for e in extents) + self.assertEqual(total, IMAGE_SIZE) + + def test_write_flush_read(self): + data = randbytes(1001, IMAGE_SIZE) + resp = http_put(f"{self._url}?flush=y", data) + body = json.loads(resp.read()) + self.assertTrue(body["flushed"]) + + resp2 = http_get(self._url) + self.assertEqual(resp2.read(), data) + + +class TestWriteThenReadFile(ImageServerTestCase): + def setUp(self): + self._tid, self._url, self._path, self._cleanup = make_file_transfer() + + def tearDown(self): + self._cleanup() + + def test_put_then_get_roundtrip(self): + data = randbytes(1100, IMAGE_SIZE) + http_put(self._url, data) + resp = http_get(self._url) + self.assertEqual(resp.read(), data) + + +class TestRegisterUseUnregisterUse(ImageServerTestCase): + def test_unregistered_transfer_returns_404(self): + data = randbytes(1200, IMAGE_SIZE) + tid, url, path, cleanup = make_file_transfer(data=data) + + resp = http_get(url) + self.assertEqual(resp.read(), data) + + cleanup() + + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(url) + self.assertEqual(ctx.exception.code, 404) + + +class TestMultipleTransfersSimultaneous(ImageServerTestCase): + @test_timeout(120) + def test_operate_on_file_and_nbd_concurrently(self): + file_data = randbytes(1300, IMAGE_SIZE) + nbd_data = randbytes(1301, IMAGE_SIZE) + + ftid, furl, fpath, fcleanup = make_file_transfer(data=file_data) + ntid, nurl, nbd_server, ncleanup = make_nbd_transfer() + + try: + log.info("Writing data to NBD transfer %s", ntid) + http_put(nurl, nbd_data) + + log.info("Starting concurrent file + NBD reads") + with ThreadPoolExecutor(max_workers=4) as pool: + f_file = pool.submit(_fetch, furl) + f_nbd = pool.submit(_fetch, nurl) + + self.assertEqual(f_file.result(timeout=FUTURES_TIMEOUT), file_data) + self.assertEqual(f_nbd.result(timeout=FUTURES_TIMEOUT), nbd_data) + log.info("Concurrent reads completed successfully") + finally: + fcleanup() + ncleanup() + + +class TestLargeChunkedTransfer(ImageServerTestCase): + def test_put_larger_than_chunk_size_file(self): + """Upload data that spans multiple CHUNK_SIZE boundaries.""" + tid, url, path, cleanup = make_file_transfer() + try: + data = randbytes(1400, IMAGE_SIZE) + http_put(url, data) + resp = http_get(url) + self.assertEqual(resp.read(), data) + finally: + cleanup() + + def test_nbd_put_larger_than_chunk_size(self): + tid, url, nbd_server, cleanup = make_nbd_transfer() + try: + data = randbytes(1401, IMAGE_SIZE) + http_put(url, data) + resp = http_get(url) + self.assertEqual(resp.read(), data) + finally: + cleanup() + + +class TestEdgeCases(ImageServerTestCase): + def test_get_not_found_path(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(f"{self.base_url}/not/images/path") + self.assertEqual(ctx.exception.code, 404) + + def test_post_unknown_tail(self): + tid, url, path, cleanup = make_file_transfer() + try: + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_post(f"{url}/unknown") + self.assertEqual(ctx.exception.code, 404) + finally: + cleanup() + + def test_get_extents_then_flush_nbd(self): + tid, url, nbd_server, cleanup = make_nbd_transfer() + try: + http_put(url, randbytes(1500, IMAGE_SIZE)) + + resp = http_get(f"{url}/extents") + self.assertEqual(resp.status, 200) + resp.read() + + resp2 = http_post(f"{url}/flush") + body = json.loads(resp2.read()) + self.assertTrue(body["ok"]) + finally: + cleanup() + + +if __name__ == "__main__": + try: + unittest.main() + finally: + shutdown_image_server() diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_control_socket.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_control_socket.py new file mode 100644 index 000000000000..187592ff1070 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_control_socket.py @@ -0,0 +1,258 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Unix domain control socket protocol (register / unregister / status).""" + +import json +import socket +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .test_base import ImageServerTestCase, make_tmp_image, shutdown_image_server, test_timeout + + +class TestStatus(ImageServerTestCase): + def test_status_returns_ok(self): + resp = self.ctrl({"action": "status"}) + self.assertEqual(resp["status"], "ok") + self.assertIn("active_transfers", resp) + + def test_status_count_is_integer(self): + resp = self.ctrl({"action": "status"}) + self.assertIsInstance(resp["active_transfers"], int) + self.assertGreaterEqual(resp["active_transfers"], 0) + + +class TestRegister(ImageServerTestCase): + def test_register_file_backend(self): + img = make_tmp_image() + tid = f"test-{uuid.uuid4().hex[:8]}" + try: + resp = self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "ok") + self.assertGreaterEqual(resp["active_transfers"], 1) + finally: + self.ctrl({"action": "unregister", "transfer_id": tid}) + + def test_register_nbd_backend(self): + tid = f"test-{uuid.uuid4().hex[:8]}" + try: + resp = self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "nbd", "socket": "/tmp/fake.sock"}, + }) + self.assertEqual(resp["status"], "ok") + finally: + self.ctrl({"action": "unregister", "transfer_id": tid}) + + def test_register_increments_active_count(self): + img = make_tmp_image() + before = self.ctrl({"action": "status"})["active_transfers"] + tid = f"test-{uuid.uuid4().hex[:8]}" + try: + self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + after = self.ctrl({"action": "status"})["active_transfers"] + self.assertEqual(after, before + 1) + finally: + self.ctrl({"action": "unregister", "transfer_id": tid}) + + def test_register_missing_transfer_id(self): + img = make_tmp_image() + resp = self.ctrl({ + "action": "register", + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_empty_transfer_id(self): + img = make_tmp_image() + resp = self.ctrl({ + "action": "register", + "transfer_id": "", + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_missing_config(self): + resp = self.ctrl({ + "action": "register", + "transfer_id": f"test-{uuid.uuid4().hex[:8]}", + }) + self.assertEqual(resp["status"], "error") + + def test_register_invalid_backend(self): + resp = self.ctrl({ + "action": "register", + "transfer_id": f"test-{uuid.uuid4().hex[:8]}", + "config": {"backend": "invalid"}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_file_missing_path(self): + resp = self.ctrl({ + "action": "register", + "transfer_id": f"test-{uuid.uuid4().hex[:8]}", + "config": {"backend": "file"}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_nbd_missing_socket(self): + resp = self.ctrl({ + "action": "register", + "transfer_id": f"test-{uuid.uuid4().hex[:8]}", + "config": {"backend": "nbd"}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_path_traversal_rejected(self): + img = make_tmp_image() + resp = self.ctrl({ + "action": "register", + "transfer_id": "../etc/passwd", + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_dot_rejected(self): + img = make_tmp_image() + resp = self.ctrl({ + "action": "register", + "transfer_id": ".", + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_slash_rejected(self): + img = make_tmp_image() + resp = self.ctrl({ + "action": "register", + "transfer_id": "a/b", + "config": {"backend": "file", "file": img}, + }) + self.assertEqual(resp["status"], "error") + + def test_register_duplicate_replaces(self): + img = make_tmp_image() + tid = f"test-{uuid.uuid4().hex[:8]}" + try: + self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + count_before = self.ctrl({"action": "status"})["active_transfers"] + self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + count_after = self.ctrl({"action": "status"})["active_transfers"] + self.assertEqual(count_after, count_before) + finally: + self.ctrl({"action": "unregister", "transfer_id": tid}) + + +class TestUnregister(ImageServerTestCase): + def test_unregister_existing(self): + img = make_tmp_image() + tid = f"test-{uuid.uuid4().hex[:8]}" + self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + before = self.ctrl({"action": "status"})["active_transfers"] + resp = self.ctrl({"action": "unregister", "transfer_id": tid}) + self.assertEqual(resp["status"], "ok") + self.assertEqual(resp["active_transfers"], before - 1) + + def test_unregister_nonexistent(self): + resp = self.ctrl({"action": "unregister", "transfer_id": "does-not-exist"}) + self.assertEqual(resp["status"], "ok") + + def test_unregister_missing_id(self): + resp = self.ctrl({"action": "unregister"}) + self.assertEqual(resp["status"], "error") + + +class TestUnknownAction(ImageServerTestCase): + def test_unknown_action(self): + resp = self.ctrl({"action": "foobar"}) + self.assertEqual(resp["status"], "error") + self.assertIn("unknown", resp.get("message", "").lower()) + + +class TestMalformed(ImageServerTestCase): + def test_malformed_json(self): + sock_path = self.server["ctrl_sock"] + payload = b"not valid json\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(5) + s.connect(sock_path) + s.sendall(payload) + s.shutdown(socket.SHUT_WR) + data = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + resp = json.loads(data.strip()) + self.assertEqual(resp["status"], "error") + + +class TestConcurrentRegistrations(ImageServerTestCase): + @test_timeout(60) + def test_concurrent_registers(self): + img = make_tmp_image() + tids = [f"conc-{uuid.uuid4().hex[:8]}" for _ in range(20)] + results = [] + + def register_one(tid): + return self.ctrl({ + "action": "register", + "transfer_id": tid, + "config": {"backend": "file", "file": img}, + }) + + try: + with ThreadPoolExecutor(max_workers=10) as pool: + futures = {pool.submit(register_one, tid): tid for tid in tids} + for f in as_completed(futures, timeout=30): + results.append(f.result()) + + self.assertTrue(all(r["status"] == "ok" for r in results)) + finally: + for tid in tids: + self.ctrl({"action": "unregister", "transfer_id": tid}) + + +if __name__ == "__main__": + try: + unittest.main() + finally: + shutdown_image_server() diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_file_backend.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_file_backend.py new file mode 100644 index 000000000000..be6eb259cc38 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_file_backend.py @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for HTTP operations against a file-backend transfer.""" + +import json +import os +import unittest +import urllib.error + +from .test_base import ( + IMAGE_SIZE, + ImageServerTestCase, + http_get, + http_options, + http_patch, + http_post, + http_put, + make_file_transfer, + randbytes, + shutdown_image_server, +) + + +class FileBackendTestCase(ImageServerTestCase): + """Base that creates a file-backend transfer per test.""" + + def setUp(self): + self._tid, self._url, self._path, self._cleanup = make_file_transfer() + + def tearDown(self): + self._cleanup() + + +class TestOptions(FileBackendTestCase): + def test_options_returns_features(self): + resp = http_options(self._url) + self.assertEqual(resp.status, 200) + body = json.loads(resp.read()) + self.assertIn("flush", body["features"]) + self.assertGreaterEqual(body["max_readers"], 1) + self.assertGreaterEqual(body["max_writers"], 1) + + def test_options_allowed_methods(self): + resp = http_options(self._url) + methods = resp.getheader("Access-Control-Allow-Methods") + for m in ("GET", "PUT", "POST", "OPTIONS"): + self.assertIn(m, methods) + + +class TestGetFull(FileBackendTestCase): + def test_get_full_returns_file_content(self): + with open(self._path, "rb") as f: + expected = f.read() + resp = http_get(self._url) + self.assertEqual(resp.status, 200) + data = resp.read() + self.assertEqual(len(data), len(expected)) + self.assertEqual(data, expected) + + def test_get_full_content_type(self): + resp = http_get(self._url) + resp.read() + self.assertIn("application/octet-stream", resp.getheader("Content-Type")) + + def test_get_full_content_length(self): + resp = http_get(self._url) + resp.read() + self.assertEqual(int(resp.getheader("Content-Length")), os.path.getsize(self._path)) + + +class TestGetRange(FileBackendTestCase): + def test_get_range_partial(self): + with open(self._path, "rb") as f: + f.seek(100) + expected = f.read(200) + resp = http_get(self._url, headers={"Range": "bytes=100-299"}) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.read(), expected) + + def test_get_range_content_range_header(self): + size = os.path.getsize(self._path) + resp = http_get(self._url, headers={"Range": "bytes=0-99"}) + self.assertEqual(resp.status, 206) + resp.read() + self.assertEqual(resp.getheader("Content-Range"), f"bytes 0-99/{size}") + + def test_get_range_suffix(self): + with open(self._path, "rb") as f: + expected = f.read()[-100:] + resp = http_get(self._url, headers={"Range": "bytes=-100"}) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.read(), expected) + + def test_get_range_open_ended(self): + with open(self._path, "rb") as f: + f.seek(IMAGE_SIZE - 50) + expected = f.read() + resp = http_get(self._url, headers={"Range": f"bytes={IMAGE_SIZE - 50}-"}) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.read(), expected) + + def test_get_range_unsatisfiable(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(self._url, headers={"Range": f"bytes={IMAGE_SIZE + 100}-{IMAGE_SIZE + 200}"}) + self.assertEqual(ctx.exception.code, 416) + + +class TestPut(FileBackendTestCase): + def test_put_full_upload(self): + new_data = randbytes(99, IMAGE_SIZE) + resp = http_put(self._url, new_data) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + self.assertEqual(body["bytes_written"], IMAGE_SIZE) + + with open(self._path, "rb") as f: + self.assertEqual(f.read(), new_data) + + def test_put_with_flush(self): + new_data = randbytes(100, IMAGE_SIZE) + resp = http_put(f"{self._url}?flush=y", new_data) + body = json.loads(resp.read()) + self.assertTrue(body["ok"]) + self.assertTrue(body["flushed"]) + + def test_put_verify_by_get(self): + new_data = randbytes(101, IMAGE_SIZE) + http_put(self._url, new_data) + resp = http_get(self._url) + self.assertEqual(resp.read(), new_data) + + def test_put_with_content_range_rejected(self): + data = b"x" * 100 + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_put(self._url, data, headers={"Content-Range": "bytes 0-99/*"}) + self.assertEqual(ctx.exception.code, 400) + + def test_put_with_range_header_rejected(self): + data = b"x" * 100 + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_put(self._url, data, headers={"Range": "bytes=0-99"}) + self.assertEqual(ctx.exception.code, 400) + + +class TestFlush(FileBackendTestCase): + def test_post_flush(self): + resp = http_post(f"{self._url}/flush") + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + + +class TestPatchRejected(FileBackendTestCase): + def test_patch_rejected_for_file(self): + data = json.dumps({"op": "zero", "size": 100}).encode() + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_patch(self._url, data, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(data)), + }) + self.assertEqual(ctx.exception.code, 400) + + +class TestExtentsRejected(FileBackendTestCase): + def test_extents_rejected_for_file(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(f"{self._url}/extents") + self.assertEqual(ctx.exception.code, 400) + + +class TestUnknownImage(ImageServerTestCase): + def test_get_unknown_image(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(f"{self.base_url}/images/nonexistent-id") + self.assertEqual(ctx.exception.code, 404) + + def test_put_unknown_image(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_put(f"{self.base_url}/images/nonexistent-id", b"data") + self.assertEqual(ctx.exception.code, 404) + + def test_options_unknown_image(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_options(f"{self.base_url}/images/nonexistent-id") + self.assertEqual(ctx.exception.code, 404) + + +class TestRoundTrip(FileBackendTestCase): + def test_put_then_get_roundtrip(self): + payload = randbytes(200, IMAGE_SIZE) + http_put(self._url, payload) + resp = http_get(self._url) + self.assertEqual(resp.read(), payload) + + def test_put_then_ranged_get_roundtrip(self): + payload = randbytes(201, IMAGE_SIZE) + http_put(self._url, payload) + resp = http_get(self._url, headers={"Range": "bytes=512-1023"}) + self.assertEqual(resp.read(), payload[512:1024]) + + def test_multiple_puts_last_wins(self): + first = randbytes(300, IMAGE_SIZE) + second = randbytes(301, IMAGE_SIZE) + http_put(self._url, first) + http_put(self._url, second) + resp = http_get(self._url) + self.assertEqual(resp.read(), second) + + +if __name__ == "__main__": + try: + unittest.main() + finally: + shutdown_image_server() diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py new file mode 100644 index 000000000000..4c0e66003b37 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py @@ -0,0 +1,393 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for HTTP operations against an NBD-backend transfer (real qemu-nbd).""" + +import json +import unittest +import urllib.error +import urllib.request + +from .test_base import ( + IMAGE_SIZE, + ImageServerTestCase, + http_get, + http_options, + http_patch, + http_post, + http_put, + make_nbd_transfer, + randbytes, + shutdown_image_server, +) + + +class NbdBackendTestCase(ImageServerTestCase): + """Base that creates an NBD-backend transfer per test.""" + + def setUp(self): + self._tid, self._url, self._nbd, self._cleanup = make_nbd_transfer() + + def tearDown(self): + self._cleanup() + + +class TestOptions(NbdBackendTestCase): + def test_options_returns_extents_feature(self): + resp = http_options(self._url) + self.assertEqual(resp.status, 200) + body = json.loads(resp.read()) + self.assertIn("extents", body["features"]) + + def test_options_includes_patch_method(self): + resp = http_options(self._url) + methods = resp.getheader("Access-Control-Allow-Methods") + self.assertIn("PATCH", methods) + + def test_options_has_capabilities(self): + resp = http_options(self._url) + body = json.loads(resp.read()) + self.assertGreaterEqual(body["max_readers"], 1) + self.assertGreaterEqual(body["max_writers"], 1) + + +class TestGetFull(NbdBackendTestCase): + def test_get_full_returns_image_data(self): + with open(self._nbd.image_path, "rb") as f: + expected = f.read() + resp = http_get(self._url) + data = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(len(data), len(expected)) + self.assertEqual(data, expected) + + def test_get_full_content_length(self): + resp = http_get(self._url) + resp.read() + self.assertEqual(int(resp.getheader("Content-Length")), IMAGE_SIZE) + + +class TestGetRange(NbdBackendTestCase): + def test_get_range_partial(self): + test_data = randbytes(50, IMAGE_SIZE) + http_put(self._url, test_data) + + resp = http_get(self._url, headers={"Range": "bytes=100-299"}) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.read(), test_data[100:300]) + + def test_get_range_content_range_header(self): + resp = http_get(self._url, headers={"Range": "bytes=0-99"}) + self.assertEqual(resp.status, 206) + resp.read() + self.assertEqual(resp.getheader("Content-Range"), f"bytes 0-99/{IMAGE_SIZE}") + + def test_get_range_suffix(self): + test_data = randbytes(51, IMAGE_SIZE) + http_put(self._url, test_data) + + resp = http_get(self._url, headers={"Range": "bytes=-100"}) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.read(), test_data[-100:]) + + def test_get_range_unsatisfiable(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_get(self._url, headers={"Range": f"bytes={IMAGE_SIZE + 100}-{IMAGE_SIZE + 200}"}) + self.assertEqual(ctx.exception.code, 416) + + +class TestPutFull(NbdBackendTestCase): + def test_put_full_upload(self): + new_data = randbytes(60, IMAGE_SIZE) + resp = http_put(self._url, new_data) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + self.assertEqual(body["bytes_written"], IMAGE_SIZE) + + resp2 = http_get(self._url) + self.assertEqual(resp2.read(), new_data) + + def test_put_with_flush(self): + new_data = randbytes(61, IMAGE_SIZE) + resp = http_put(f"{self._url}?flush=y", new_data) + body = json.loads(resp.read()) + self.assertTrue(body["ok"]) + self.assertTrue(body["flushed"]) + + +class TestPutRange(NbdBackendTestCase): + def test_put_content_range(self): + base_data = randbytes(70, IMAGE_SIZE) + http_put(self._url, base_data) + + patch_data = b"\xAB" * 512 + resp = http_put(self._url, patch_data, headers={ + "Content-Range": "bytes 0-511/*", + "Content-Length": str(len(patch_data)), + }) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + self.assertEqual(body["bytes_written"], 512) + + resp2 = http_get(self._url, headers={"Range": "bytes=0-511"}) + self.assertEqual(resp2.read(), patch_data) + + resp3 = http_get(self._url, headers={"Range": "bytes=512-1023"}) + self.assertEqual(resp3.read(), base_data[512:1024]) + + def test_put_content_range_with_flush(self): + base_data = b"\x00" * IMAGE_SIZE + http_put(self._url, base_data) + + patch_data = b"\xFF" * 256 + resp = http_put(f"{self._url}?flush=y", patch_data, headers={ + "Content-Range": "bytes 1024-1279/*", + "Content-Length": str(len(patch_data)), + }) + body = json.loads(resp.read()) + self.assertTrue(body["ok"]) + self.assertTrue(body["flushed"]) + + +class TestPatchRange(NbdBackendTestCase): + def test_patch_binary_range(self): + base_data = randbytes(80, IMAGE_SIZE) + http_put(self._url, base_data) + + patch_data = b"\xCD" * 1024 + resp = http_patch(self._url, patch_data, headers={ + "Range": "bytes=2048-3071", + "Content-Type": "application/octet-stream", + "Content-Length": str(len(patch_data)), + }) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + self.assertEqual(body["bytes_written"], 1024) + + resp2 = http_get(self._url, headers={"Range": "bytes=2048-3071"}) + self.assertEqual(resp2.read(), patch_data) + + def test_patch_multiple_ranges_preserves_unwritten(self): + base_data = randbytes(81, IMAGE_SIZE) + http_put(self._url, base_data) + + patch1 = b"\x11" * 256 + http_patch(self._url, patch1, headers={ + "Range": "bytes=0-255", + "Content-Type": "application/octet-stream", + "Content-Length": "256", + }) + + patch2 = b"\x22" * 256 + http_patch(self._url, patch2, headers={ + "Range": "bytes=512-767", + "Content-Type": "application/octet-stream", + "Content-Length": "256", + }) + + resp = http_get(self._url, headers={"Range": "bytes=0-767"}) + got = resp.read() + self.assertEqual(got[:256], patch1) + self.assertEqual(got[256:512], base_data[256:512]) + self.assertEqual(got[512:768], patch2) + + +class TestPatchZero(NbdBackendTestCase): + def test_patch_zero(self): + data = randbytes(90, IMAGE_SIZE) + http_put(self._url, data) + + payload = json.dumps({"op": "zero", "size": 4096, "offset": 0}).encode() + resp = http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + + resp2 = http_get(self._url, headers={"Range": "bytes=0-4095"}) + self.assertEqual(resp2.read(), b"\x00" * 4096) + + def test_patch_zero_with_flush(self): + data = b"\xFF" * IMAGE_SIZE + http_put(self._url, data) + + payload = json.dumps({"op": "zero", "size": 512, "offset": 1024, "flush": True}).encode() + resp = http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + body = json.loads(resp.read()) + self.assertTrue(body["ok"]) + + resp2 = http_get(self._url, headers={"Range": "bytes=1024-1535"}) + self.assertEqual(resp2.read(), b"\x00" * 512) + + def test_patch_zero_preserves_neighbors(self): + data = randbytes(91, IMAGE_SIZE) + http_put(self._url, data) + + payload = json.dumps({"op": "zero", "size": 256, "offset": 512}).encode() + http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + + resp = http_get(self._url, headers={"Range": "bytes=0-1023"}) + got = resp.read() + self.assertEqual(got[:512], data[:512]) + self.assertEqual(got[512:768], b"\x00" * 256) + self.assertEqual(got[768:1024], data[768:1024]) + + +class TestPatchFlush(NbdBackendTestCase): + def test_patch_flush_op(self): + payload = json.dumps({"op": "flush"}).encode() + resp = http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + + +class TestPostFlush(NbdBackendTestCase): + def test_post_flush(self): + resp = http_post(f"{self._url}/flush") + body = json.loads(resp.read()) + self.assertEqual(resp.status, 200) + self.assertTrue(body["ok"]) + + +class TestExtents(NbdBackendTestCase): + def test_get_allocation_extents(self): + resp = http_get(f"{self._url}/extents") + self.assertEqual(resp.status, 200) + extents = json.loads(resp.read()) + self.assertIsInstance(extents, list) + self.assertGreaterEqual(len(extents), 1) + for ext in extents: + self.assertIn("start", ext) + self.assertIn("length", ext) + self.assertIn("zero", ext) + + def test_extents_cover_full_image(self): + resp = http_get(f"{self._url}/extents") + extents = json.loads(resp.read()) + total = sum(e["length"] for e in extents) + self.assertEqual(total, IMAGE_SIZE) + + def test_extents_dirty_context_without_bitmap(self): + resp = http_get(f"{self._url}/extents?context=dirty") + self.assertEqual(resp.status, 200) + extents = json.loads(resp.read()) + self.assertIsInstance(extents, list) + self.assertGreaterEqual(len(extents), 1) + for ext in extents: + self.assertIn("dirty", ext) + self.assertTrue(ext["dirty"]) + + def test_extents_after_write_and_zero(self): + http_put(self._url, randbytes(95, IMAGE_SIZE)) + + payload = json.dumps({"op": "zero", "size": 4096, "offset": 0}).encode() + http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + + resp = http_get(f"{self._url}/extents") + extents = json.loads(resp.read()) + self.assertGreaterEqual(len(extents), 1) + total = sum(e["length"] for e in extents) + self.assertEqual(total, IMAGE_SIZE) + + +class TestErrorCases(NbdBackendTestCase): + def test_patch_unsupported_op(self): + payload = json.dumps({"op": "invalid"}).encode() + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + self.assertEqual(ctx.exception.code, 400) + + def test_patch_zero_missing_size(self): + payload = json.dumps({"op": "zero", "offset": 0}).encode() + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_patch(self._url, payload, headers={ + "Content-Type": "application/json", + "Content-Length": str(len(payload)), + }) + self.assertEqual(ctx.exception.code, 400) + + def test_put_missing_content_length(self): + import http.client + from urllib.parse import urlparse + parsed = urlparse(self._url) + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=30) + try: + conn.putrequest("PUT", parsed.path) + conn.endheaders() + resp = conn.getresponse() + self.assertEqual(resp.status, 400) + finally: + conn.close() + + +class TestRoundTrip(NbdBackendTestCase): + def test_write_read_full_roundtrip(self): + payload = randbytes(110, IMAGE_SIZE) + http_put(self._url, payload) + resp = http_get(self._url) + self.assertEqual(resp.read(), payload) + + def test_write_read_range_roundtrip(self): + payload = randbytes(111, IMAGE_SIZE) + http_put(self._url, payload) + + for start, end in [(0, 255), (1024, 2047), (IMAGE_SIZE - 512, IMAGE_SIZE - 1)]: + resp = http_get(self._url, headers={"Range": f"bytes={start}-{end}"}) + self.assertEqual(resp.read(), payload[start:end + 1]) + + def test_range_write_read_roundtrip(self): + http_put(self._url, b"\x00" * IMAGE_SIZE) + + chunk = randbytes(112, 4096) + http_put(self._url, chunk, headers={ + "Content-Range": "bytes 8192-12287/*", + "Content-Length": str(len(chunk)), + }) + + resp = http_get(self._url, headers={"Range": "bytes=8192-12287"}) + self.assertEqual(resp.read(), chunk) + + resp2 = http_get(self._url, headers={"Range": "bytes=0-4095"}) + self.assertEqual(resp2.read(), b"\x00" * 4096) + + +if __name__ == "__main__": + try: + unittest.main() + finally: + shutdown_image_server() From bb213dcdcba80f753667c605b7b0e222b3e19ea7 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 23 Mar 2026 21:18:28 +0530 Subject: [PATCH 073/129] extract constants used in image server --- .../kvm/imageserver/backends/nbd.py | 12 ++++++++--- .../hypervisor/kvm/imageserver/constants.py | 17 ++++++++++++++- .../vm/hypervisor/kvm/imageserver/handler.py | 4 ++-- .../vm/hypervisor/kvm/imageserver/server.py | 21 +++++++++++++------ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py index ed6d3ac6ed7a..48ba1d9fe906 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py @@ -21,7 +21,13 @@ import nbd -from ..constants import CHUNK_SIZE, NBD_STATE_DIRTY, NBD_STATE_HOLE, NBD_STATE_ZERO +from ..constants import ( + CHUNK_SIZE, + NBD_BLOCK_STATUS_CHUNK, + NBD_STATE_DIRTY, + NBD_STATE_HOLE, + NBD_STATE_ZERO, +) from ..util import merge_dirty_zero_extents from .base import BackendSession, ImageBackend @@ -175,7 +181,7 @@ def get_allocation_extents(self) -> List[Dict[str, Any]]: return [{"start": 0, "length": size, "zero": False}] allocation_extents: List[Dict[str, Any]] = [] - chunk = min(size, 64 * 1024 * 1024) + chunk = min(size, NBD_BLOCK_STATUS_CHUNK) offset = 0 def extent_cb(*args: Any, **kwargs: Any) -> int: @@ -246,7 +252,7 @@ def get_extents_dirty_and_zero( allocation_extents: List[Tuple[int, int, bool]] = [] dirty_extents: List[Tuple[int, int, bool]] = [] - chunk = min(size, 64 * 1024 * 1024) + chunk = min(size, NBD_BLOCK_STATUS_CHUNK) offset = 0 def extent_cb(*args: Any, **kwargs: Any) -> int: diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py index 4e8d5c86da5b..6e0ae03a0b58 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/constants.py +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -26,5 +26,20 @@ MAX_PARALLEL_READS = 8 MAX_PARALLEL_WRITES = 1 -CFG_DIR = "/tmp/imagetransfer" +# HTTP server defaults +DEFAULT_LISTEN_ADDRESS = "127.0.0.1" +DEFAULT_HTTP_PORT = 54323 + +# Control socket CONTROL_SOCKET = "/var/run/cloudstack/image-server.sock" +CONTROL_SOCKET_BACKLOG = 32 +CONTROL_SOCKET_PERMISSIONS = 0o660 +CONTROL_RECV_BUFFER = 4096 + +# Maximum size of a JSON body in a PATCH request (zero / flush ops) +MAX_PATCH_JSON_SIZE = 64 * 1024 # 64 KiB + +# Byte range requested per block_status call for NBD extent queries +NBD_BLOCK_STATUS_CHUNK = 64 * 1024 * 1024 # 64 MiB + +CFG_DIR = "/tmp/imagetransfer" diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index a689467238bd..9bfed8d52f93 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -26,7 +26,7 @@ from .backends import NbdBackend, create_backend from .concurrency import ConcurrencyManager from .config import TransferRegistry -from .constants import CHUNK_SIZE, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES +from .constants import CHUNK_SIZE, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES, MAX_PATCH_JSON_SIZE from .util import is_fallback_dirty_response, json_bytes, now_s @@ -422,7 +422,7 @@ def do_PATCH(self) -> None: except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") return - if content_length <= 0 or content_length > 64 * 1024: + if content_length <= 0 or content_length > MAX_PATCH_JSON_SIZE: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") return diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index d348bf4950db..53d4383b96f1 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -33,7 +33,16 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] from .concurrency import ConcurrencyManager from .config import TransferRegistry -from .constants import CONTROL_SOCKET, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES +from .constants import ( + CONTROL_RECV_BUFFER, + CONTROL_SOCKET, + CONTROL_SOCKET_BACKLOG, + CONTROL_SOCKET_PERMISSIONS, + DEFAULT_HTTP_PORT, + DEFAULT_LISTEN_ADDRESS, + MAX_PARALLEL_READS, + MAX_PARALLEL_WRITES, +) from .handler import Handler @@ -95,7 +104,7 @@ def _handle_control_conn(conn: socket.socket, registry: TransferRegistry) -> Non try: data = b"" while True: - chunk = conn.recv(4096) + chunk = conn.recv(CONTROL_RECV_BUFFER) if not chunk: break data += chunk @@ -151,8 +160,8 @@ def _control_listener(registry: TransferRegistry, sock_path: str) -> None: srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) srv.bind(sock_path) - os.chmod(sock_path, 0o660) - srv.listen(5) + os.chmod(sock_path, CONTROL_SOCKET_PERMISSIONS) + srv.listen(CONTROL_SOCKET_BACKLOG) logging.info("control socket listening on %s", sock_path) while True: @@ -168,8 +177,8 @@ def main() -> None: parser = argparse.ArgumentParser( description="CloudStack image server backed by NBD / local file" ) - parser.add_argument("--listen", default="127.0.0.1", help="Address to bind") - parser.add_argument("--port", type=int, default=54323, help="Port to listen on") + parser.add_argument("--listen", default=DEFAULT_LISTEN_ADDRESS, help="Address to bind") + parser.add_argument("--port", type=int, default=DEFAULT_HTTP_PORT, help="Port to listen on") parser.add_argument( "--control-socket", default=CONTROL_SOCKET, From 5b7184781336b69ad78910c6d810098949a4256f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 26 Mar 2026 11:32:52 +0530 Subject: [PATCH 074/129] fix network listing Signed-off-by: Abhishek Kumar --- .../java/com/cloud/network/dao/NetworkDao.java | 2 ++ .../com/cloud/network/dao/NetworkDaoImpl.java | 10 +++++++++- .../cloudstack/veeam/adapter/ServerAdapter.java | 10 ++++------ .../com/cloud/api/query/dao/UserVmJoinDao.java | 3 +++ .../cloud/api/query/dao/UserVmJoinDaoImpl.java | 11 +++++++++++ .../com/cloud/vpc/dao/MockNetworkDaoImpl.java | 15 ++++++++++----- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index fdca6e43f00f..341f9d7cb848 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -98,6 +98,8 @@ public interface NetworkDao extends GenericDao, StateDao listByZoneAndTrafficType(long zoneId, TrafficType trafficType); + List listByTrafficType(TrafficType trafficType); + void setCheckForGc(long networkId); int getNetworkCountByNetworkOffId(long networkOfferingId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 9f7ffabac930..9a01a8ee7e3b 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -29,7 +29,6 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; import org.springframework.stereotype.Component; @@ -63,6 +62,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.SequenceFetcher; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @Component @@ -640,6 +640,14 @@ public List listByZoneAndTrafficType(final long zoneId, final Traffic return listBy(sc, null); } + @Override + public List listByTrafficType(final TrafficType trafficType) { + final SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("trafficType", trafficType); + + return listBy(sc, null); + } + @Override public int getNetworkCountByNetworkOffId(final long networkOfferingId) { final SearchCriteria sc = NetworksCount.create(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 8fe47387b93d..c957d95a2bbc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -144,6 +144,7 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.NetworkModel; +import com.cloud.network.Networks; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; @@ -465,7 +466,7 @@ public List listNetworksByDcId(final String uuid) { if (dataCenterVO == null) { throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); } - List networks = networkDao.listAll(); + List networks = networkDao.listByZoneAndTrafficType(dataCenterVO.getId(), Networks.TrafficType.Guest); return NetworkVOToNetworkConverter.toNetworkList(networks, (dcId) -> dataCenterVO); } @@ -509,7 +510,7 @@ public Network getNetwork(String uuid) { } public List listAllVnicProfiles() { - final List networks = networkDao.listAll(); + final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest); return NetworkVOToVnicProfileConverter.toVnicProfileList(networks, this::getZoneById); } @@ -522,7 +523,7 @@ public VnicProfile getVnicProfile(String uuid) { } public List listAllInstances() { - List vms = userVmJoinDao.listAll(); + List vms = userVmJoinDao.listByHypervisorType(Hypervisor.HypervisorType.KVM); return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); } @@ -996,9 +997,6 @@ public Disk createDisk(Disk request) { throw new InvalidParameterValueException("Request disk data is empty"); } String name = request.getName(); - if (StringUtils.isBlank(name) && !name.startsWith("Veeam_KvmBackupDisk_")) { - throw new InvalidParameterValueException("Only worker VM disk creation is supported"); - } if (request.getStorageDomains() == null || CollectionUtils.isEmpty(request.getStorageDomains().getItems()) || request.getStorageDomains().getItems().size() > 1) { throw new InvalidParameterValueException("Exactly one storage domain must be specified"); diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java index 79312460d2c0..351e367e8d05 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java @@ -17,6 +17,7 @@ package com.cloud.api.query.dao; import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.db.GenericDao; @@ -49,4 +50,6 @@ List listByAccountServiceOfferingTemplateAndNotInState(long accoun List listEligibleInstancesWithExpiredLease(); List listLeaseInstancesExpiringInDays(int days); + + List listByHypervisorType(Hypervisor.HypervisorType hypervisorType); } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 687fea1c4e33..39b2b9b94218 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import com.cloud.gpu.dao.VgpuProfileDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.service.dao.ServiceOfferingDao; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.annotation.AnnotationService; @@ -832,4 +833,14 @@ public List listLeaseInstancesExpiringInDays(int days) { } return listBy(sc); } + + @Override + public List listByHypervisorType(Hypervisor.HypervisorType hypervisorType) { + SearchBuilder sb = createSearchBuilder(); + sb.and("hypervisorType", sb.entity().getHypervisorType(), Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("hypervisorType", hypervisorType); + return listBy(sc); + } } diff --git a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java index 8a0bec56df7b..cf71d74498fc 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java @@ -16,6 +16,11 @@ // under the License. package com.cloud.vpc.dao; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import com.cloud.network.Network; import com.cloud.network.Network.GuestType; import com.cloud.network.Networks.TrafficType; @@ -26,11 +31,6 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - @DB() public class MockNetworkDaoImpl extends GenericDaoBase implements NetworkDao { @@ -165,6 +165,11 @@ public List listByZoneAndTrafficType(final long zoneId, final Traffic return null; } + @Override + public List listByTrafficType(final TrafficType trafficType) { + return null; + } + @Override public void setCheckForGc(final long networkId) { } From 8d42d5f186e052add0f39b43c86c24e62bf05478 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Sat, 28 Mar 2026 14:34:39 +0530 Subject: [PATCH 075/129] change name from IncrementalBackupService to KVMBackupExportService --- .../command/admin/backup/CreateImageTransferCmd.java | 6 +++--- .../command/admin/backup/DeleteVmCheckpointCmd.java | 6 +++--- .../api/command/admin/backup/FinalizeBackupCmd.java | 6 +++--- .../admin/backup/FinalizeImageTransferCmd.java | 6 +++--- .../command/admin/backup/ListImageTransfersCmd.java | 6 +++--- .../command/admin/backup/ListVmCheckpointsCmd.java | 6 +++--- .../api/command/admin/backup/StartBackupCmd.java | 8 ++++---- ...ackupService.java => KVMBackupExportService.java} | 2 +- .../cloudstack/veeam/adapter/ServerAdapter.java | 12 ++++++------ ...viceImpl.java => KVMBackupExportServiceImpl.java} | 4 ++-- .../core/spring-server-core-managers-context.xml | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) rename api/src/main/java/org/apache/cloudstack/backup/{IncrementalBackupService.java => KVMBackupExportService.java} (97%) rename server/src/main/java/org/apache/cloudstack/backup/{IncrementalBackupServiceImpl.java => KVMBackupExportServiceImpl.java} (99%) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index c50a914cd13e..a60ce02c4306 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -29,7 +29,7 @@ import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.backup.ImageTransfer; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; import com.cloud.utils.EnumUtils; @@ -42,7 +42,7 @@ public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, @@ -86,7 +86,7 @@ public ImageTransfer.Format getFormat() { @Override public void execute() { - ImageTransferResponse response = incrementalBackupService.createImageTransfer(this); + ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java index 47b62ddcc501..a39a597d4703 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -27,7 +27,7 @@ import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; @APICommand(name = "deleteVirtualMachineCheckpoint", @@ -38,7 +38,7 @@ public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, @@ -71,7 +71,7 @@ public void setCheckpointId(String checkpointId) { @Override public void execute() { - boolean result = incrementalBackupService.deleteVmCheckpoint(this); + boolean result = kvmBackupExportService.deleteVmCheckpoint(this); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java index e6e270c7f6f8..81d16bf80fea 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -31,7 +31,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; import com.cloud.event.EventTypes; @@ -44,7 +44,7 @@ public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Inject private BackupManager backupManager; @@ -73,7 +73,7 @@ public Long getBackupId() { @Override public void execute() { - Backup backup = incrementalBackupService.finalizeBackup(this); + Backup backup = kvmBackupExportService.finalizeBackup(this); if (backup == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java index b8a21a104e37..ce853fb49d2b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -27,7 +27,7 @@ import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.SuccessResponse; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; @APICommand(name = "finalizeImageTransfer", @@ -38,7 +38,7 @@ public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Parameter(name = ApiConstants.ID, type = CommandType.UUID, @@ -53,7 +53,7 @@ public Long getImageTransferId() { @Override public void execute() { - boolean result = incrementalBackupService.finalizeImageTransfer(this); + boolean result = kvmBackupExportService.finalizeImageTransfer(this); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java index 99d596312d6c..eb7fb604bc13 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -30,7 +30,7 @@ import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; @APICommand(name = "listImageTransfers", @@ -41,7 +41,7 @@ public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Parameter(name = ApiConstants.ID, type = CommandType.UUID, @@ -65,7 +65,7 @@ public Long getBackupId() { @Override public void execute() { - List responses = incrementalBackupService.listImageTransfers(this); + List responses = kvmBackupExportService.listImageTransfers(this); ListResponse response = new ListResponse<>(); response.setResponses(responses); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java index 0d223ffaf5db..208d791006a7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -30,7 +30,7 @@ import org.apache.cloudstack.api.response.CheckpointResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; @APICommand(name = "listVirtualMachineCheckpoints", description = "List checkpoints for a VM", @@ -40,7 +40,7 @@ public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, @@ -55,7 +55,7 @@ public Long getVmId() { @Override public void execute() { - List responses = incrementalBackupService.listVmCheckpoints(this); + List responses = kvmBackupExportService.listVmCheckpoints(this); ListResponse response = new ListResponse<>(); response.setResponses(responses); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java index b3a87178d164..04ebfe143cce 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -31,7 +31,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; import com.cloud.event.EventTypes; @@ -44,7 +44,7 @@ public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { @Inject - private IncrementalBackupService incrementalBackupService; + private KVMBackupExportService kvmBackupExportService; @Inject private BackupManager backupManager; @@ -81,7 +81,7 @@ public String getDescription() { @Override public void execute() { try { - Backup backup = incrementalBackupService.startBackup(this); + Backup backup = kvmBackupExportService.startBackup(this); BackupResponse response = backupManager.createBackupResponse(backup, null); response.setResponseName(getCommandName()); @@ -98,7 +98,7 @@ public long getEntityOwnerId() { @Override public void create() { - Backup backup = incrementalBackupService.createBackup(this); + Backup backup = kvmBackupExportService.createBackup(this); if (backup != null) { setEntityId(backup.getId()); diff --git a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java similarity index 97% rename from api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java rename to api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index 053f1c1455e0..cddd316b8671 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -36,7 +36,7 @@ /** * Service for managing oVirt-style incremental backups using libvirt checkpoints */ -public interface IncrementalBackupService extends Configurable, PluggableService { +public interface KVMBackupExportService extends Configurable, PluggableService { ConfigKey ImageTransferPollingInterval = new ConfigKey<>("Advanced", Long.class, "image.transfer.polling.interval", diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index c957d95a2bbc..a0eed5dbfc14 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -69,7 +69,7 @@ import org.apache.cloudstack.backup.ImageTransfer.Direction; import org.apache.cloudstack.backup.ImageTransfer.Format; import org.apache.cloudstack.backup.ImageTransferVO; -import org.apache.cloudstack.backup.IncrementalBackupService; +import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.context.CallContext; @@ -263,7 +263,7 @@ public class ServerAdapter extends ManagerBase { ImageTransferDao imageTransferDao; @Inject - IncrementalBackupService incrementalBackupService; + KVMBackupExportService kvmBackupExportService; @Inject QueryService queryService; @@ -1212,7 +1212,7 @@ public boolean cancelImageTransfer(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } - return incrementalBackupService.cancelImageTransfer(vo.getId()); + return kvmBackupExportService.cancelImageTransfer(vo.getId()); } public boolean finalizeImageTransfer(String uuid) { @@ -1220,7 +1220,7 @@ public boolean finalizeImageTransfer(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } - return incrementalBackupService.finalizeImageTransfer(vo.getId()); + return kvmBackupExportService.finalizeImageTransfer(vo.getId()); } private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { @@ -1228,7 +1228,7 @@ private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Directio CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = - incrementalBackupService.createImageTransfer(volumeId, backupId, direction, format); + kvmBackupExportService.createImageTransfer(volumeId, backupId, direction, format); ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); } finally { @@ -1517,7 +1517,7 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { DeleteVmCheckpointCmd cmd = new DeleteVmCheckpointCmd(); ComponentContext.inject(cmd); cmd.setVmId(vo.getId()); - incrementalBackupService.deleteVmCheckpoint(cmd); + kvmBackupExportService.deleteVmCheckpoint(cmd); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); } finally { diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java similarity index 99% rename from server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java rename to server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index dd4dc7565955..a69ce2fd7e5f 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -76,7 +76,7 @@ import com.cloud.vm.dao.VMInstanceDao; @Component -public class IncrementalBackupServiceImpl extends ManagerBase implements IncrementalBackupService { +public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService { @Inject private VMInstanceDao vmInstanceDao; @@ -874,7 +874,7 @@ private long getVolumeTotalSize(VolumeVO volume) { @Override public String getConfigComponentName() { - return IncrementalBackupService.class.getSimpleName(); + return KVMBackupExportService.class.getSimpleName(); } @Override diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index a8c51fdc77e3..48fe5bb415df 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -347,7 +347,7 @@ - + From b6d480cfb1861e821d7d0506ba500ff7e206c9a9 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Sat, 28 Mar 2026 15:47:04 +0530 Subject: [PATCH 076/129] hide kvm backup export service apis behind a global config --- .../admin/backup/CreateImageTransferCmd.java | 4 ++-- .../admin/backup/DeleteVmCheckpointCmd.java | 4 ++-- .../command/admin/backup/FinalizeBackupCmd.java | 4 ++-- .../admin/backup/FinalizeImageTransferCmd.java | 4 ++-- .../admin/backup/ListImageTransfersCmd.java | 4 ++-- .../admin/backup/ListVmCheckpointsCmd.java | 4 ++-- .../command/admin/backup/StartBackupCmd.java | 4 ++-- .../backup/KVMBackupExportService.java | 4 ++++ .../backup/KVMBackupExportServiceImpl.java | 17 +++++++++-------- 9 files changed, 27 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index a60ce02c4306..8948d1a0d5fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -35,9 +35,9 @@ import com.cloud.utils.EnumUtils; @APICommand(name = "createImageTransfer", - description = "Create image transfer for a disk in backup", + description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.", responseObject = ImageTransferResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java index a39a597d4703..d0e17e86d427 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -31,9 +31,9 @@ import org.apache.cloudstack.context.CallContext; @APICommand(name = "deleteVirtualMachineCheckpoint", - description = "Delete a VM checkpoint", + description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.", responseObject = SuccessResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java index 81d16bf80fea..45173f8668ee 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -37,9 +37,9 @@ import com.cloud.event.EventTypes; @APICommand(name = "finalizeBackup", - description = "Finalize a VM backup session", + description = "Finalize a VM backup session. This API is intended for testing only and is disabled by default.", responseObject = BackupResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java index ce853fb49d2b..d483f78b4228 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -31,9 +31,9 @@ import org.apache.cloudstack.context.CallContext; @APICommand(name = "finalizeImageTransfer", - description = "Finalize an image transfer", + description = "Finalize an image transfe. This API is intended for testing only and is disabled by default.r", responseObject = SuccessResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java index eb7fb604bc13..2565ef241a6b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -34,9 +34,9 @@ import org.apache.cloudstack.context.CallContext; @APICommand(name = "listImageTransfers", - description = "List image transfers for a backup", + description = "List image transfers for a backup. This API is intended for testing only and is disabled by default.", responseObject = ImageTransferResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java index 208d791006a7..a61661e982de 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -33,9 +33,9 @@ import org.apache.cloudstack.backup.KVMBackupExportService; @APICommand(name = "listVirtualMachineCheckpoints", - description = "List checkpoints for a VM", + description = "List checkpoints for a VM. This API is intended for testing only and is disabled by default.", responseObject = CheckpointResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java index 04ebfe143cce..a5c4773c0fc4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -37,9 +37,9 @@ import com.cloud.event.EventTypes; @APICommand(name = "startBackup", - description = "Start a VM backup session (oVirt-style incremental backup)", + description = "Start a VM backup session. This API is intended for testing only and is disabled by default.", responseObject = BackupResponse.class, - since = "4.22.0", + since = "4.23.0", authorized = {RoleType.Admin}) public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index cddd316b8671..6093293779b1 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -43,6 +43,10 @@ public interface KVMBackupExportService extends Configurable, PluggableService { "10", "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Hidden", Boolean.class, + "expose.kvm.backup.export.service.apis", + "false", + "Enable to expose APIs for testing the KVM Backup Export Service.", false, ConfigKey.Scope.Global); /** * Creates a backup session for a VM */ diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index a69ce2fd7e5f..5ff82362a799 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -123,7 +123,6 @@ private boolean isDummyOffering(Long backupOfferingId) { @Override public Backup createBackup(StartBackupCmd cmd) { - //ToDo: add config check, access check, resource count check, etc. Long vmId = cmd.getVmId(); VMInstanceVO vm = vmInstanceDao.findById(vmId); @@ -705,13 +704,15 @@ public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { @Override public List> getCommands() { List> cmdList = new ArrayList<>(); - cmdList.add(StartBackupCmd.class); - cmdList.add(FinalizeBackupCmd.class); - cmdList.add(CreateImageTransferCmd.class); - cmdList.add(FinalizeImageTransferCmd.class); - cmdList.add(ListImageTransfersCmd.class); - cmdList.add(ListVmCheckpointsCmd.class); - cmdList.add(DeleteVmCheckpointCmd.class); + if (ExposeKVMBackupExportServiceApis.value()) { + cmdList.add(StartBackupCmd.class); + cmdList.add(FinalizeBackupCmd.class); + cmdList.add(CreateImageTransferCmd.class); + cmdList.add(FinalizeImageTransferCmd.class); + cmdList.add(ListImageTransfersCmd.class); + cmdList.add(ListVmCheckpointsCmd.class); + cmdList.add(DeleteVmCheckpointCmd.class); + } return cmdList; } From ca0ad93d61860276770940f2b918efffa471f70d Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Sat, 28 Mar 2026 17:47:39 +0530 Subject: [PATCH 077/129] Remove dependency on backup offering. Make backup export service exclusive to other backup providers. --- .../cloudstack/backup/BackupManager.java | 4 +- .../META-INF/db/schema-42210to42300.sql | 2 +- .../cloudstack/backup/BackupManagerImpl.java | 6 +- .../backup/KVMBackupExportServiceImpl.java | 83 +++++++------------ 4 files changed, 36 insertions(+), 59 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 6c0121a3e4d8..e2016f76c1f7 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -58,7 +58,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer ConfigKey BackupProviderPlugin = new ValidatedConfigKey<>("Advanced", String.class, "backup.framework.provider.plugin", "dummy", - "The backup and recovery provider plugin. Valid plugin values: dummy, veeam, networker and nas", + "The backup and recovery provider plugin. Valid plugin values: dummy, veeam, networker, nas and veeam-kvm", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key(), value -> validateBackupProviderConfig((String)value)); ConfigKey BackupSyncPollingInterval = new ConfigKey<>("Advanced", Long.class, @@ -263,7 +263,7 @@ static void validateBackupProviderConfig(String value) { if (value != null && (value.contains(",") || value.trim().contains(" "))) { throw new IllegalArgumentException("Multiple backup provider plugins are not supported. Please provide a single plugin value."); } - List validPlugins = List.of("dummy", "veeam", "networker", "nas"); + List validPlugins = List.of("dummy", "veeam", "networker", "nas", "veeam-kvm"); if (value != null && !validPlugins.contains(value)) { throw new IllegalArgumentException("Invalid backup provider plugin: " + value + ". Valid plugin values are: " + String.join(", ", validPlugins)); } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index b0063bff53e0..90f8d1d61eb1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -120,7 +120,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NUL -- Add checkpoint tracking fields to backups table for incremental backup support CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for this backup session"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for the next incremental backup"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 7ff345960f8c..ac5476a2e122 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -2411,8 +2411,10 @@ public BackupResponse createBackupResponse(Backup backup, Boolean listVmDetails) backedUpVolumes = new Gson().toJson(backup.getBackedUpVolumes().toArray(), Backup.VolumeInfo[].class); } response.setVolumes(backedUpVolumes); - response.setBackupOfferingId(offering.getUuid()); - response.setBackupOffering(offering.getName()); + if (offering != null) { + response.setBackupOfferingId(offering.getUuid()); + response.setBackupOffering(offering.getName()); + } response.setAccountId(account.getUuid()); response.setAccount(account.getAccountName()); response.setDomainId(domain.getUuid()); diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 5ff82362a799..37ae291107f8 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -41,7 +41,6 @@ import org.apache.cloudstack.api.response.CheckpointResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; @@ -75,6 +74,8 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; +import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; + @Component public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService { @@ -96,9 +97,6 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject private AgentManager agentManager; - @Inject - private BackupOfferingDao backupOfferingDao; - @Inject private HostDao hostDao; @@ -107,20 +105,6 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup private Timer imageTransferTimer; - private boolean isDummyOffering(Long backupOfferingId) { - if (backupOfferingId == null) { - throw new CloudRuntimeException("VM not assigned a backup offering"); - } - BackupOfferingVO offering = backupOfferingDao.findById(backupOfferingId); - if (offering == null) { - throw new CloudRuntimeException("Backup offering not found: " + backupOfferingId); - } - if ("dummy".equalsIgnoreCase(offering.getName())) { - return true; - } - return false; - } - @Override public Backup createBackup(StartBackupCmd cmd) { Long vmId = cmd.getVmId(); @@ -130,12 +114,13 @@ public Backup createBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("VM not found: " + vmId); } - if (vm.getState() != State.Running && vm.getState() != State.Stopped) { - throw new CloudRuntimeException("VM must be running or stopped to start backup"); + if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(vm.getDataCenterId()))) { + throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + + " to \"veeam-kvm\" to enable the feature."); } - if (vm.getBackupOfferingId() == null) { - throw new CloudRuntimeException("VM not assigned a backup offering"); + if (vm.getState() != State.Running && vm.getState() != State.Stopped) { + throw new CloudRuntimeException("VM must be running or stopped to start backup"); } Backup existingBackup = backupDao.findByVmId(vmId); @@ -158,7 +143,7 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setStatus(Backup.Status.Queued); - backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setBackupOfferingId(0L); backup.setDate(new Date()); String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); @@ -175,7 +160,7 @@ public Backup createBackup(StartBackupCmd cmd) { return backupDao.persist(backup); } - protected void removedFailedBackup(BackupVO backup) { + protected void removeFailedBackup(BackupVO backup) { backup.setStatus(Backup.Status.Error); backupDao.update(backup.getId(), backup); backupDao.remove(backup.getId()); @@ -208,23 +193,17 @@ public Backup startBackup(StartBackupCmd cmd) { vm.getState() == State.Stopped ); - boolean dummyOffering = isDummyOffering(vm.getBackupOfferingId()); - StartBackupAnswer answer; try { - if (dummyOffering) { - answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis()); - } else { - answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); - } + answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { - removedFailedBackup(backup); + removeFailedBackup(backup); logger.error("Failed to communicate with agent on {} for {} start", host, backup, e); throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { - removedFailedBackup(backup); + removeFailedBackup(backup); logger.error("Failed to start {} due to: {}", backup, answer.getDetails()); throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); } @@ -264,8 +243,6 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { throw new CloudRuntimeException("VM not found: " + vmId); } - boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); - updateBackupState(backup, Backup.Status.FinalizingTransfer); List transfers = imageTransferDao.listByBackupId(backupId); @@ -282,12 +259,7 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { StopBackupAnswer answer; try { - if (dummyOffering) { - answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); - } else { - answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); - } - + answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { updateBackupState(backup, Backup.Status.Failed); throw new CloudRuntimeException("Failed to communicate with agent", e); @@ -325,7 +297,6 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu if (backup == null) { throw new CloudRuntimeException("Backup not found: " + backupId); } - boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); if (ImageTransfer.Backend.file.equals(backend)) { throw new CloudRuntimeException("File backend is not supported for download"); } @@ -349,11 +320,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu try { CreateImageTransferAnswer answer; - if (dummyOffering) { - answer = new CreateImageTransferAnswer(transferCmd, true, "Dummy answer", "image-transfer-id", "nbd://127.0.0.1:10809/vda"); - } else { - answer = (CreateImageTransferAnswer) agentManager.send(backup.getHostId(), transferCmd); - } + answer = (CreateImageTransferAnswer) agentManager.send(backup.getHostId(), transferCmd); if (!answer.getResult()) { throw new CloudRuntimeException("Failed to create image transfer: " + answer.getDetails()); @@ -525,6 +492,15 @@ public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTran ImageTransfer imageTransfer; VolumeVO volume = volumeDao.findById(volumeId); + if (volume == null) { + throw new CloudRuntimeException("Volume not found with the specified Id"); + } + + if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(volume.getDataCenterId()))) { + throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + + " to \"veeam-kvm\" to enable the feature."); + } + ImageTransferVO existingTransfer = imageTransferDao.findUnfinishedByVolume(volume.getId()); if (existingTransfer != null) { throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid()); @@ -558,16 +534,10 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId); BackupVO backup = backupDao.findById(imageTransfer.getBackupId()); - boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId()); Answer answer; try { - if (dummyOffering) { - answer = new Answer(finalizeCmd, true, "Image transfer finalized."); - } else { - answer = agentManager.send(backup.getHostId(), finalizeCmd); - } - + answer = agentManager.send(backup.getHostId(), finalizeCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -695,6 +665,11 @@ public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { if (vm == null) { throw new CloudRuntimeException("VM not found: " + cmd.getVmId()); } + if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(vm.getDataCenterId()))) { + throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + + " to \"veeam-kvm\" to enable the feature."); + } + vm.setActiveCheckpointId(null); vm.setActiveCheckpointCreateTime(null); vmInstanceDao.update(cmd.getVmId(), vm); From ce19b922e6b258d547fb4fea6c309ab85e1b5120 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 26 Mar 2026 11:55:42 +0530 Subject: [PATCH 078/129] coalesce similar extents --- .../kvm/imageserver/backends/nbd.py | 4 +- .../hypervisor/kvm/imageserver/concurrency.py | 4 +- .../vm/hypervisor/kvm/imageserver/server.py | 2 +- .../kvm/imageserver/tests/test_util.py | 122 ++++++++++++++++++ scripts/vm/hypervisor/kvm/imageserver/util.py | 48 ++++++- 5 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_util.py diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py index 48ba1d9fe906..aa247be29f21 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py @@ -28,7 +28,7 @@ NBD_STATE_HOLE, NBD_STATE_ZERO, ) -from ..util import merge_dirty_zero_extents +from ..util import coalesce_allocation_extents, merge_dirty_zero_extents from .base import BackendSession, ImageBackend @@ -225,7 +225,7 @@ def extent_cb(*args: Any, **kwargs: Any) -> int: return [{"start": 0, "length": size, "zero": False}] if not allocation_extents: return [{"start": 0, "length": size, "zero": False}] - return allocation_extents + return coalesce_allocation_extents(allocation_extents) def get_extents_dirty_and_zero( self, dirty_bitmap_context: str diff --git a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py index a446786224d8..7d91aea60131 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py +++ b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py @@ -29,8 +29,8 @@ class ConcurrencyManager: """ Manages per-image read/write semaphores and per-image mutual-exclusion locks. - Each image_id gets its own independent pool of read slots (default 8) - and write slots (default 1), so concurrent transfers to different images + Each image_id gets its own independent pool of read slots (default MAX_PARALLEL_READS) + and write slots (default MAX_PARALLEL_WRITES), so concurrent transfers to different images do not contend with each other. The per-image lock serialises operations that must not overlap on the diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index 53d4383b96f1..6d6648030d1d 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -54,7 +54,7 @@ def make_handler( Create a Handler subclass with injected dependencies. BaseHTTPRequestHandler is instantiated per-request by the server, so we - cannot pass constructor args. Instead we set class-level attributes. + cannot pass constructor args. Instead, we set class-level attributes. """ class ConfiguredHandler(Handler): diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_util.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_util.py new file mode 100644 index 000000000000..159dff30a929 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_util.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for imageserver.util extent coalescing helpers.""" + +import unittest + +from imageserver.util import ( + coalesce_allocation_extents, + coalesce_dirty_zero_extents, + merge_dirty_zero_extents, +) + + +class TestCoalesceAllocationExtents(unittest.TestCase): + def test_empty(self): + self.assertEqual(coalesce_allocation_extents([]), []) + + def test_single(self): + inp = [{"start": 0, "length": 4096, "zero": False}] + out = coalesce_allocation_extents(inp) + self.assertEqual(out, [{"start": 0, "length": 4096, "zero": False}]) + self.assertIsNot(out[0], inp[0]) + + def test_merges_contiguous_same_zero(self): + inp = [ + {"start": 0, "length": 10, "zero": False}, + {"start": 10, "length": 5, "zero": False}, + {"start": 15, "length": 100, "zero": False}, + ] + self.assertEqual( + coalesce_allocation_extents(inp), + [{"start": 0, "length": 115, "zero": False}], + ) + + def test_does_not_merge_different_zero(self): + inp = [ + {"start": 0, "length": 64, "zero": False}, + {"start": 64, "length": 64, "zero": True}, + {"start": 128, "length": 64, "zero": False}, + ] + self.assertEqual(coalesce_allocation_extents(inp), inp) + + def test_does_not_merge_gap(self): + inp = [ + {"start": 0, "length": 100, "zero": False}, + {"start": 200, "length": 50, "zero": False}, + ] + self.assertEqual(coalesce_allocation_extents(inp), inp) + + def test_does_not_merge_same_zero_with_gap(self): + inp = [ + {"start": 0, "length": 10, "zero": True}, + {"start": 20, "length": 10, "zero": True}, + ] + self.assertEqual(coalesce_allocation_extents(inp), inp) + + +class TestCoalesceDirtyZeroExtents(unittest.TestCase): + def test_empty(self): + self.assertEqual(coalesce_dirty_zero_extents([]), []) + + def test_single(self): + inp = [{"start": 0, "length": 8192, "dirty": True, "zero": False}] + out = coalesce_dirty_zero_extents(inp) + self.assertEqual( + out, [{"start": 0, "length": 8192, "dirty": True, "zero": False}] + ) + + def test_merges_contiguous_same_flags(self): + inp = [ + {"start": 0, "length": 50, "dirty": True, "zero": False}, + {"start": 50, "length": 50, "dirty": True, "zero": False}, + ] + self.assertEqual( + coalesce_dirty_zero_extents(inp), + [{"start": 0, "length": 100, "dirty": True, "zero": False}], + ) + + def test_does_not_merge_differing_dirty(self): + inp = [ + {"start": 0, "length": 32, "dirty": False, "zero": False}, + {"start": 32, "length": 32, "dirty": True, "zero": False}, + ] + self.assertEqual(coalesce_dirty_zero_extents(inp), inp) + + def test_does_not_merge_differing_zero(self): + inp = [ + {"start": 0, "length": 16, "dirty": False, "zero": False}, + {"start": 16, "length": 16, "dirty": False, "zero": True}, + ] + self.assertEqual(coalesce_dirty_zero_extents(inp), inp) + + +class TestMergeDirtyZeroExtentsCoalescing(unittest.TestCase): + def test_coalesces_adjacent_identical_flags_after_boundary_merge(self): + """Boundary grid can split one logical run; coalesce should reunite.""" + allocation = [(0, 200, False)] + dirty = [(0, 100, False), (100, 100, False)] + merged = merge_dirty_zero_extents(allocation, dirty, 200) + self.assertEqual( + merged, + [{"start": 0, "length": 200, "dirty": False, "zero": False}], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/vm/hypervisor/kvm/imageserver/util.py b/scripts/vm/hypervisor/kvm/imageserver/util.py index 71e51cec65a1..473f58a50c07 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/util.py +++ b/scripts/vm/hypervisor/kvm/imageserver/util.py @@ -20,6 +20,52 @@ from typing import Any, Dict, List, Set, Tuple +def coalesce_allocation_extents( + extents: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Merge contiguous extents that share the same ``zero`` flag.""" + if not extents: + return [] + out: List[Dict[str, Any]] = [dict(extents[0])] + for e in extents[1:]: + prev = out[-1] + if ( + prev["start"] + prev["length"] == e["start"] + and prev["zero"] == e["zero"] + ): + prev["length"] += e["length"] + else: + out.append({"start": e["start"], "length": e["length"], "zero": e["zero"]}) + return out + + +def coalesce_dirty_zero_extents( + extents: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Merge contiguous extents that share the same ``dirty`` and ``zero`` flags.""" + if not extents: + return [] + out: List[Dict[str, Any]] = [dict(extents[0])] + for e in extents[1:]: + prev = out[-1] + if ( + prev["start"] + prev["length"] == e["start"] + and prev["dirty"] == e["dirty"] + and prev["zero"] == e["zero"] + ): + prev["length"] += e["length"] + else: + out.append( + { + "start": e["start"], + "length": e["length"], + "dirty": e["dirty"], + "zero": e["zero"], + } + ) + return out + + def json_bytes(obj: Any) -> bytes: return json.dumps(obj, separators=(",", ":"), ensure_ascii=False).encode("utf-8") @@ -63,7 +109,7 @@ def lookup( "zero": lookup(allocation_extents, a, False), } ) - return result + return coalesce_dirty_zero_extents(result) def is_fallback_dirty_response(extents: List[Dict[str, Any]]) -> bool: From 9a7008a86e92ecf1780357f08d3d2a9477214fed Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Sat, 28 Mar 2026 18:35:47 +0530 Subject: [PATCH 079/129] change image server default port from 54323 to 54322 --- .../cloud/hypervisor/kvm/resource/LibvirtComputingResource.java | 2 ++ .../wrapper/LibvirtCreateImageTransferCommandWrapper.java | 2 +- .../wrapper/LibvirtFinalizeImageTransferCommandWrapper.java | 2 +- scripts/vm/hypervisor/kvm/imageserver/__init__.py | 2 +- scripts/vm/hypervisor/kvm/imageserver/constants.py | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 821be05cfb21..34f166a7fb69 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -382,6 +382,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; + public static final int IMAGE_SERVER_DEFAULT_PORT = 54322; + protected int qcow2DeltaMergeTimeout; private String modifyVlanPath; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 71beafe9fa1b..859b7498859c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -135,7 +135,7 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r } } - final int imageServerPort = 54323; + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; if (!startImageServerIfNotRunning(imageServerPort, resource)) { return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java index 3f0026ae38f2..240dc3622192 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -41,7 +41,7 @@ private void resetService(String unitName) { private boolean stopImageServer() { String unitName = "cloudstack-image-server"; - final int imageServerPort = 54323; + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); diff --git a/scripts/vm/hypervisor/kvm/imageserver/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/__init__.py index 69eec98956a3..dc9505310395 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/__init__.py +++ b/scripts/vm/hypervisor/kvm/imageserver/__init__.py @@ -31,7 +31,7 @@ Usage:: # As a module - python -m imageserver --listen 127.0.0.1 --port 54323 + python -m imageserver --listen 127.0.0.1 --port 54322 # Or via the systemd service started by createImageTransfer """ diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py index 6e0ae03a0b58..33cf3001d7a1 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/constants.py +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -28,7 +28,7 @@ # HTTP server defaults DEFAULT_LISTEN_ADDRESS = "127.0.0.1" -DEFAULT_HTTP_PORT = 54323 +DEFAULT_HTTP_PORT = 54322 # Control socket CONTROL_SOCKET = "/var/run/cloudstack/image-server.sock" From 2bbbcae3a983761020189211b9b7c7e69df8ae81 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 29 Mar 2026 07:34:59 +0530 Subject: [PATCH 080/129] Add tests for qcow2 file parallel range reads and puts --- .../kvm/imageserver/tests/test_base.py | 191 +++++++++-- .../kvm/imageserver/tests/test_nbd_backend.py | 313 ++++++++++++++++++ 2 files changed, 470 insertions(+), 34 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py index 91e7eda79ed4..c322a9920477 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py @@ -20,19 +20,21 @@ Provides: - A singleton image server process started once for the entire test run. -- Control-socket helpers using pure-Python AF_UNIX (no socat). +- Server stdout/stderr appended to ``/imageserver.log``. +- On shutdown: stop the child process, close the log handle, unlink the control socket; + the temp directory and ``imageserver.log`` are left on disk. +- Control-socket helpers using pure-Python AF_UNIX. - qemu-nbd server management. - Transfer registration / teardown helpers. - HTTP helper functions. """ +import atexit import functools import json import logging import os import random -import select -import shutil import signal import socket import subprocess @@ -42,7 +44,7 @@ import unittest import uuid from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, TextIO IMAGE_SIZE = 1 * 1024 * 1024 # 1 MiB SERVER_STARTUP_TIMEOUT = 10 @@ -87,6 +89,9 @@ def _alarm(signum, frame): _tmp_dir: Optional[str] = None _server_proc: Optional[subprocess.Popen] = None _server_info: Optional[Dict[str, Any]] = None +_server_log_fp: Optional[TextIO] = None +_server_log_path: Optional[str] = None +_atexit_registered: bool = False def _free_port() -> int: @@ -158,9 +163,21 @@ def get_tmp_dir() -> str: return _tmp_dir +def _read_log_tail(path: str, max_bytes: int = 65536) -> str: + """Return up to *max_bytes* of UTF-8 text from the end of *path*.""" + try: + with open(path, "rb") as f: + f.seek(0, os.SEEK_END) + size = f.tell() + f.seek(max(0, size - max_bytes)) + return f.read().decode("utf-8", errors="replace") + except OSError as e: + return f"(could not read log: {e})" + + def get_image_server() -> Dict[str, Any]: """Return the singleton image-server info dict, starting it if needed.""" - global _server_proc, _server_info + global _server_proc, _server_info, _server_log_fp, _server_log_path, _atexit_registered if _server_info is not None: return _server_info @@ -168,6 +185,8 @@ def get_image_server() -> Dict[str, Any]: tmp = get_tmp_dir() port = _free_port() ctrl_sock = os.path.join(tmp, "ctrl.sock") + log_path = os.path.join(tmp, "imageserver.log") + _server_log_path = log_path imageserver_pkg = str(Path(__file__).resolve().parent.parent) parent_dir = str(Path(imageserver_pkg).parent) @@ -175,6 +194,17 @@ def get_image_server() -> Dict[str, Any]: env = os.environ.copy() env["PYTHONPATH"] = parent_dir + os.pathsep + env.get("PYTHONPATH", "") + _server_log_fp = open( + log_path, "a", encoding="utf-8", buffering=1, errors="replace" + ) + try: + _server_log_fp.write( + "\n========== imageserver test subprocess log ==========\n" + ) + _server_log_fp.flush() + except OSError: + pass + proc = subprocess.Popen( [ sys.executable, "-m", "imageserver", @@ -184,8 +214,8 @@ def get_image_server() -> Dict[str, Any]: ], cwd=parent_dir, env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=_server_log_fp, + stderr=_server_log_fp, ) _server_proc = proc @@ -193,9 +223,26 @@ def get_image_server() -> Dict[str, Any]: _wait_for_control_socket(ctrl_sock) except RuntimeError: proc.kill() - stdout, stderr = proc.communicate(timeout=5) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + try: + _server_log_fp.flush() + except OSError: + pass + tail = _read_log_tail(log_path) + try: + _server_log_fp.close() + except OSError: + pass + _server_log_fp = None + _server_proc = None raise RuntimeError( - f"Image server failed to start.\nstdout: {stdout.decode()}\nstderr: {stderr.decode()}" + "Image server failed to start.\n" + f"Log file: {log_path}\n" + f"--- log tail ---\n{tail}" ) def send(msg: dict) -> dict: @@ -206,19 +253,25 @@ def send(msg: dict) -> dict: "port": port, "ctrl_sock": ctrl_sock, "send": send, + "imageserver_log": log_path, } + if not _atexit_registered: + atexit.register(shutdown_image_server) + _atexit_registered = True + sys.stdout.write( + "\n[IMAGESERVER_TEST] child image server log file: %s\n\n" % log_path + ) + sys.stdout.flush() return _server_info def shutdown_image_server() -> None: - global _server_proc, _server_info, _tmp_dir + global _server_proc, _server_info, _tmp_dir, _server_log_fp, _server_log_path + ctrl_sock: Optional[str] = None + if _server_info is not None: + ctrl_sock = _server_info.get("ctrl_sock") + if _server_proc is not None: - for pipe in (_server_proc.stdout, _server_proc.stderr): - if pipe: - try: - pipe.close() - except Exception: - pass _server_proc.terminate() try: _server_proc.wait(timeout=5) @@ -226,33 +279,59 @@ def shutdown_image_server() -> None: _server_proc.kill() _server_proc.wait(timeout=5) _server_proc = None + if _server_log_fp is not None: + try: + _server_log_fp.flush() + _server_log_fp.close() + except OSError: + pass + _server_log_fp = None _server_info = None - if _tmp_dir is not None: - shutil.rmtree(_tmp_dir, ignore_errors=True) - _tmp_dir = None + _server_log_path = None + + if ctrl_sock: + try: + os.unlink(ctrl_sock) + except FileNotFoundError: + pass + + # Leave temp dir and imageserver.log on disk for debugging; clear pointer only. + _tmp_dir = None # ── qemu-nbd server ──────────────────────────────────────────────────── class QemuNbdServer: - """Manages a qemu-nbd process exporting a raw image over a Unix socket.""" - - def __init__(self, image_path: str, socket_path: str, image_size: int = IMAGE_SIZE): + """Manages a qemu-nbd process exporting a disk image over a Unix socket.""" + + def __init__( + self, + image_path: str, + socket_path: str, + image_size: int = IMAGE_SIZE, + image_format: str = "raw", + ): self.image_path = image_path self.socket_path = socket_path self.image_size = image_size + self.image_format = image_format self._proc: Optional[subprocess.Popen] = None def start(self) -> None: if not os.path.exists(self.image_path): - with open(self.image_path, "wb") as f: - f.truncate(self.image_size) + if self.image_format == "raw": + with open(self.image_path, "wb") as f: + f.truncate(self.image_size) + else: + raise FileNotFoundError( + f"disk image not found for format {self.image_format!r}: {self.image_path}" + ) self._proc = subprocess.Popen( [ "qemu-nbd", "--socket", self.socket_path, - "--format", "raw", + "--format", self.image_format, "--persistent", "--shared=8", "--cache=none", @@ -355,6 +434,43 @@ def cleanup(): return transfer_id, url, server, cleanup +def make_nbd_transfer_existing_disk(image_path: str, image_format: str = "qcow2"): + """ + Start qemu-nbd for an existing on-disk image (e.g. qcow2) and register a transfer. + + Does not delete *image_path* on cleanup (only the Unix socket under tmp). + + Returns (transfer_id, url, QemuNbdServer, cleanup_callable). + """ + srv = get_image_server() + tmp = get_tmp_dir() + sock_path = os.path.join(tmp, f"nbd_{uuid.uuid4().hex[:8]}.sock") + + server = QemuNbdServer( + image_path, sock_path, image_format=image_format + ) + server.start() + + transfer_id = f"nbd-{uuid.uuid4().hex[:8]}" + resp = srv["send"]({ + "action": "register", + "transfer_id": transfer_id, + "config": {"backend": "nbd", "socket": sock_path}, + }) + assert resp["status"] == "ok", f"register failed: {resp}" + url = f"{srv['base_url']}/images/{transfer_id}" + + def cleanup(): + srv["send"]({"action": "unregister", "transfer_id": transfer_id}) + server.stop() + try: + os.unlink(sock_path) + except FileNotFoundError: + pass + + return transfer_id, url, server, cleanup + + # ── HTTP helpers ──────────────────────────────────────────────────────── import urllib.request @@ -425,16 +541,23 @@ def _register_nbd_transfer(self): return make_nbd_transfer() @staticmethod - def dump_server_logs(): - """Read any available server stderr and print it for post-mortem debugging.""" - if _server_proc is None or _server_proc.stderr is None: + def dump_server_logs(max_bytes: int = 256 * 1024): + """Print a tail of the image-server log file (shared by all tests in the run).""" + path = _server_log_path + if not path or not os.path.isfile(path): return try: - if select.select([_server_proc.stderr], [], [], 0)[0]: - data = _server_proc.stderr.read1(64 * 1024) - if data: - sys.stderr.write("\n=== IMAGE SERVER STDERR ===\n") - sys.stderr.write(data.decode(errors="replace")) - sys.stderr.write("\n=== END SERVER STDERR ===\n") + if _server_log_fp is not None: + _server_log_fp.flush() + except OSError: + pass + try: + data = _read_log_tail(path, max_bytes=max_bytes) + if data.strip(): + sys.stderr.write("\n=== IMAGE SERVER LOG (tail) ===\n") + sys.stderr.write(data) + if not data.endswith("\n"): + sys.stderr.write("\n") + sys.stderr.write("=== END IMAGE SERVER LOG ===\n") except Exception: pass diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py index 4c0e66003b37..da120ae6bad5 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py @@ -18,19 +18,27 @@ """Tests for HTTP operations against an NBD-backend transfer (real qemu-nbd).""" import json +import os +import subprocess import unittest +import uuid import urllib.error import urllib.request +from concurrent.futures import ThreadPoolExecutor + +from imageserver.constants import MAX_PARALLEL_READS, MAX_PARALLEL_WRITES from .test_base import ( IMAGE_SIZE, ImageServerTestCase, + get_tmp_dir, http_get, http_options, http_patch, http_post, http_put, make_nbd_transfer, + make_nbd_transfer_existing_disk, randbytes, shutdown_image_server, ) @@ -322,6 +330,311 @@ def test_extents_after_write_and_zero(self): self.assertEqual(total, IMAGE_SIZE) +def _allocated_subranges(extents, granularity): + """Split each non-hole extent (zero=False) into [start, end] inclusive byte ranges.""" + out = [] + for ext in extents: + if ext.get("zero"): + continue + start = int(ext["start"]) + length = int(ext["length"]) + pos = start + end_abs = start + length + while pos < end_abs: + chunk_end = min(pos + granularity, end_abs) + out.append((pos, chunk_end - 1)) + pos = chunk_end + return out + + +def _qemu_img_virtual_size(path: str) -> int: + """Return virtual size in bytes (requires ``qemu-img`` on PATH).""" + # stdout=PIPE + universal_newlines: Python 3.6 compatible (no capture_output/text). + cp = subprocess.run( + ["qemu-img", "info", "--output=json", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + return int(json.loads(cp.stdout)["virtual-size"]) + + +def _http_error_detail(exc: urllib.error.HTTPError) -> str: + """Build a readable message from an ``HTTPError`` (status, url, JSON/text body).""" + parts = ["HTTP %s %r" % (exc.code, exc.reason), "url=%r" % getattr(exc, "url", "")] + try: + if exc.fp is not None: + raw = exc.fp.read() + if raw: + text = raw.decode("utf-8", errors="replace") + parts.append("response_body=%r" % (text,)) + except Exception as read_err: + parts.append("read_body_error=%r" % (read_err,)) + return "; ".join(parts) + + +def _http_get_checked( + url, + headers=None, + expected_status=200, + label="GET", +): + """ + Like ``http_get`` but raises ``AssertionError`` with ``_http_error_detail`` on failure. + """ + try: + resp = http_get(url, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError( + "%s failed for %r: %s" % (label, url, _http_error_detail(e)) + ) from e + if resp.status != expected_status: + body = resp.read() + raise AssertionError( + "%s %r: expected HTTP %s, got %s; body=%r" + % (label, url, expected_status, resp.status, body) + ) + return resp + + +def _http_put_checked(url, data, headers, label="PUT"): + try: + resp = http_put(url, data, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError( + "%s failed for %r: %s" % (label, url, _http_error_detail(e)) + ) from e + body = resp.read() + if resp.status != 200: + raise AssertionError( + "%s %r: expected HTTP 200, got %s; body=%r" + % (label, url, resp.status, body) + ) + return resp, body + + +def _http_post_checked(url, data=b"", headers=None, label="POST"): + try: + resp = http_post(url, data=data, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError( + "%s failed for %r: %s" % (label, url, _http_error_detail(e)) + ) from e + body = resp.read() + if resp.status != 200: + raise AssertionError( + "%s %r: expected HTTP 200, got %s; body=%r" + % (label, url, resp.status, body) + ) + return resp, body + + +class TestQcow2ExtentsParallelReads(ImageServerTestCase): + """ + Optional integration tests: export a user-supplied qcow2 via qemu-nbd, fetch + allocation extents, parallel range GETs over allocated regions, and (second + test) per-range GET-then-PUT pipeline with ``min(MAX_PARALLEL_READS, + MAX_PARALLEL_WRITES)`` workers. + + Requires ``qemu-img`` and ``qemu-nbd`` on PATH. + + Set IMAGESERVER_TEST_QCOW2 to the absolute path of a qcow2 file. + Optional: IMAGESERVER_TEST_QCOW2_READ_GRANULARITY — byte step (default 4 MiB). + """ + + def setUp(self): + super().setUp() + self._qcow2_path = os.environ.get("IMAGESERVER_TEST_QCOW2", "").strip() + if not self._qcow2_path or not os.path.isfile(self._qcow2_path): + self.skipTest( + "Set IMAGESERVER_TEST_QCOW2 to an existing qcow2 path to run this test" + ) + raw_g = os.environ.get("IMAGESERVER_TEST_QCOW2_READ_GRANULARITY", "").strip() + self._read_granularity = int(raw_g) if raw_g else 4 * 1024 * 1024 + if self._read_granularity <= 0: + self.skipTest("IMAGESERVER_TEST_QCOW2_READ_GRANULARITY must be positive") + + def test_parallel_range_reads_allocated_extents(self): + _, url, _, cleanup = make_nbd_transfer_existing_disk( + self._qcow2_path, "qcow2" + ) + try: + resp = _http_get_checked( + "%s/extents" % (url,), + expected_status=200, + label="GET /extents", + ) + extents = json.loads(resp.read()) + self.assertIsInstance(extents, list) + ranges = _allocated_subranges(extents, self._read_granularity) + if not ranges: + self.skipTest("no allocated extents (all holes/zero) in qcow2") + + def fetch(span): + start_b, end_b = span + range_hdr = "bytes=%s-%s" % (start_b, end_b) + r = _http_get_checked( + url, + headers={"Range": range_hdr}, + expected_status=206, + label="Range GET %s" % (range_hdr,), + ) + data = r.read() + expected_len = end_b - start_b + 1 + if len(data) != expected_len: + raise AssertionError( + "Range GET %s: got %d bytes, expected %d (url=%r)" + % (range_hdr, len(data), expected_len, url) + ) + + with ThreadPoolExecutor(max_workers=MAX_PARALLEL_READS) as pool: + pool.map(fetch, ranges) + finally: + cleanup() + + def test_parallel_reads_then_put_range_copy_matches_source(self): + """ + Create an empty qcow2 with the same virtual size as the source, copy every + allocated range using one worker pool: for each span, Range GET from src + then Content-Range PUT to dest. + Worker count is ``min(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES)`` so each + worker holds at most one chunk. + """ + src_path = self._qcow2_path + try: + vsize = _qemu_img_virtual_size(src_path) + except (FileNotFoundError, subprocess.CalledProcessError, KeyError, json.JSONDecodeError, TypeError, ValueError) as e: + self.skipTest(f"qemu-img info failed: {e}") + + tmp = get_tmp_dir() + dest_path = os.path.join(tmp, f"qcow2_copy_{uuid.uuid4().hex[:8]}.qcow2") + try: + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", dest_path, str(vsize)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + self.skipTest(f"qemu-img create failed: {e}") + + _, src_url, _, cleanup_src = make_nbd_transfer_existing_disk( + src_path, "qcow2" + ) + _, dest_url, _, cleanup_dest = make_nbd_transfer_existing_disk( + dest_path, "qcow2" + ) + try: + resp = _http_get_checked( + "%s/extents" % (src_url,), + expected_status=200, + label="GET src /extents", + ) + extents = json.loads(resp.read()) + ranges = _allocated_subranges(extents, self._read_granularity) + if not ranges: + self.skipTest("no allocated extents (all holes/zero) in qcow2") + + transfer_workers = max( + 1, min(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES) + ) + + def transfer_span(span): + start_b, end_b = span + range_hdr = "bytes=%s-%s" % (start_b, end_b) + r = _http_get_checked( + src_url, + headers={"Range": range_hdr}, + expected_status=206, + label="Range GET src %s" % (range_hdr,), + ) + data = r.read() + expected_len = end_b - start_b + 1 + if len(data) != expected_len: + raise AssertionError( + "Range GET src %s: got %d bytes, expected %d (url=%r)" + % (range_hdr, len(data), expected_len, src_url) + ) + end_inclusive = start_b + len(data) - 1 + cr = "bytes %s-%s/*" % (start_b, end_inclusive) + _put_resp, put_body = _http_put_checked( + dest_url, + data, + headers={ + "Content-Range": cr, + "Content-Length": str(len(data)), + }, + label="PUT dest %s" % (cr,), + ) + try: + body = json.loads(put_body) + except ValueError: + raise AssertionError( + "PUT dest %s: invalid JSON body=%r (url=%r)" + % (cr, put_body, dest_url) + ) + if not body.get("ok"): + raise AssertionError( + "PUT dest %s: JSON ok=false, full=%r (url=%r)" + % (cr, body, dest_url) + ) + if body.get("bytes_written") != len(data): + raise AssertionError( + "PUT dest %s: bytes_written=%r expected %d (url=%r)" + % (cr, body.get("bytes_written"), len(data), dest_url) + ) + + with ThreadPoolExecutor(max_workers=transfer_workers) as pool: + pool.map(transfer_span, ranges) + + _flush, flush_body = _http_post_checked( + "%s/flush" % (dest_url,), + label="POST dest /flush", + ) + try: + flush_json = json.loads(flush_body) + except ValueError: + raise AssertionError( + "POST dest /flush: invalid JSON body=%r (url=%r)" + % (flush_body, dest_url) + ) + if not flush_json.get("ok"): + raise AssertionError( + "POST dest /flush: ok=false, full=%r (url=%r)" + % (flush_json, dest_url) + ) + finally: + cleanup_dest() + cleanup_src() + + try: + cmp = subprocess.run( + ["qemu-img", "compare", src_path, dest_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + self.assertEqual( + cmp.returncode, + 0, + "qemu-img compare %r vs %r failed (rc=%s): stderr=%r stdout=%r" + % ( + src_path, + dest_path, + cmp.returncode, + cmp.stderr, + cmp.stdout, + ), + ) + finally: + try: + os.unlink(dest_path) + except FileNotFoundError: + pass + + class TestErrorCases(NbdBackendTestCase): def test_patch_unsupported_op(self): payload = json.dumps({"op": "invalid"}).encode() From ebdcf70c70c30d81ec29606da49937d311463120 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 29 Mar 2026 09:54:46 +0530 Subject: [PATCH 081/129] fix pre-commit failures --- .../apache/cloudstack/veeam/utils/JwtUtil.java | 2 +- .../veeam/api/dto/OvfXmlUtilTest.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java index c4438525c34d..a862c706b694 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java @@ -54,4 +54,4 @@ public static byte[] hmacSha256(byte[] data, byte[] key) throws Exception { mac.init(new SecretKeySpec(key, ALGORITHM)); return mac.doFinal(data); } -} \ No newline at end of file +} diff --git a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java index c01e19515fe5..bf92cc4d57fb 100644 --- a/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java +++ b/plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.cloudstack.veeam.api.dto; import static org.junit.Assert.assertEquals; From e32a6ab7d945a9085c7131574ffabe27b37d769b Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 31 Mar 2026 07:52:56 +0530 Subject: [PATCH 082/129] Make veeam-kvm exclusive with other providers finalize all pending image transfers when finalize backup is called --- .../cloudstack/backup/BackupManager.java | 4 +-- .../backup/KVMBackupExportService.java | 4 +-- .../backup/KVMBackupExportServiceImpl.java | 28 +++++++++++-------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index e2016f76c1f7..f3bd535a6b87 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -58,7 +58,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer ConfigKey BackupProviderPlugin = new ValidatedConfigKey<>("Advanced", String.class, "backup.framework.provider.plugin", "dummy", - "The backup and recovery provider plugin. Valid plugin values: dummy, veeam, networker, nas and veeam-kvm", + "The backup and recovery provider plugin. Valid plugin values: dummy, veeam, networker, nas", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key(), value -> validateBackupProviderConfig((String)value)); ConfigKey BackupSyncPollingInterval = new ConfigKey<>("Advanced", Long.class, @@ -263,7 +263,7 @@ static void validateBackupProviderConfig(String value) { if (value != null && (value.contains(",") || value.trim().contains(" "))) { throw new IllegalArgumentException("Multiple backup provider plugins are not supported. Please provide a single plugin value."); } - List validPlugins = List.of("dummy", "veeam", "networker", "nas", "veeam-kvm"); + List validPlugins = List.of("dummy", "veeam", "networker", "nas"); if (value != null && !validPlugins.contains(value)) { throw new IllegalArgumentException("Invalid backup provider plugin: " + value + ". Valid plugin values are: " + String.join(", ", validPlugins)); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index 6093293779b1..7a53c1370c6e 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -34,7 +34,7 @@ import com.cloud.utils.component.PluggableService; /** - * Service for managing oVirt-style incremental backups using libvirt checkpoints + * Service for Creating Backups and ImageTransfer sessions which will be consumed by an external orchestrator. */ public interface KVMBackupExportService extends Configurable, PluggableService { @@ -43,7 +43,7 @@ public interface KVMBackupExportService extends Configurable, PluggableService { "10", "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); - ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Hidden", Boolean.class, + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, "expose.kvm.backup.export.service.apis", "false", "Enable to expose APIs for testing the KVM Backup Export Service.", false, ConfigKey.Scope.Global); diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 37ae291107f8..4594ca6301fe 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -74,6 +74,7 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; @Component @@ -105,6 +106,10 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup private Timer imageTransferTimer; + private boolean isKVMBackupExportServiceSupported(Long zoneId) { + return !BackupFrameworkEnabled.value() || StringUtils.equals("dummy", BackupProviderPlugin.valueIn(zoneId)); + } + @Override public Backup createBackup(StartBackupCmd cmd) { Long vmId = cmd.getVmId(); @@ -114,9 +119,9 @@ public Backup createBackup(StartBackupCmd cmd) { throw new CloudRuntimeException("VM not found: " + vmId); } - if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(vm.getDataCenterId()))) { - throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + - " to \"veeam-kvm\" to enable the feature."); + if (!isKVMBackupExportServiceSupported(vm.getDataCenterId())) { + throw new CloudRuntimeException("Veeam-KVM integration can not be used along with the " + BackupProviderPlugin.valueIn(vm.getDataCenterId()) + + " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } if (vm.getState() != State.Running && vm.getState() != State.Stopped) { @@ -248,10 +253,9 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { List transfers = imageTransferDao.listByBackupId(backupId); for (ImageTransferVO transfer : transfers) { if (transfer.getPhase() != ImageTransferVO.Phase.finished) { - updateBackupState(backup, Backup.Status.Failed); - throw new CloudRuntimeException(String.format("Image transfer %s not finalized for backup: %s", transfer.getUuid(), backup.getUuid())); + logger.warn("Finalize called for backup {} while Image transfer {} is not finalized, attempting to finalize it", backup.getUuid(), transfer.getUuid()); + finalizeImageTransfer(transfer.getId()); } - imageTransferDao.remove(transfer.getId()); } if (vm.getState() == State.Running) { @@ -496,9 +500,9 @@ public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTran throw new CloudRuntimeException("Volume not found with the specified Id"); } - if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(volume.getDataCenterId()))) { - throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + - " to \"veeam-kvm\" to enable the feature."); + if (!isKVMBackupExportServiceSupported(volume.getDataCenterId())) { + throw new CloudRuntimeException("Veeam-KVM integration can not be used along with the " + BackupProviderPlugin.valueIn(volume.getDataCenterId()) + + " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } ImageTransferVO existingTransfer = imageTransferDao.findUnfinishedByVolume(volume.getId()); @@ -665,9 +669,9 @@ public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { if (vm == null) { throw new CloudRuntimeException("VM not found: " + cmd.getVmId()); } - if (!StringUtils.equals("veeam-kvm", BackupProviderPlugin.valueIn(vm.getDataCenterId()))) { - throw new CloudRuntimeException("Feature not enabled. Set Zone level config backup.framework.provider.plugin" + - " to \"veeam-kvm\" to enable the feature."); + if (!isKVMBackupExportServiceSupported(vm.getDataCenterId())) { + throw new CloudRuntimeException("Veeam-KVM integration can not be used along with the " + BackupProviderPlugin.valueIn(vm.getDataCenterId()) + + " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } vm.setActiveCheckpointId(null); From 19a8509f79c7fdf46abbeedcbff9e14524381a9f Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 31 Mar 2026 08:51:45 +0530 Subject: [PATCH 083/129] Image server TLS support --- agent/conf/agent.properties | 6 +++ .../agent/properties/AgentProperties.java | 21 ++++++++++ .../resource/LibvirtComputingResource.java | 23 +++++++++++ ...virtCreateImageTransferCommandWrapper.java | 31 ++++++++++++--- ...rtFinalizeImageTransferCommandWrapper.java | 8 ++-- .../vm/hypervisor/kvm/imageserver/server.py | 39 ++++++++++++++++++- 6 files changed, 118 insertions(+), 10 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0dc5b8211e0d..f2fcfd83eb13 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -78,6 +78,12 @@ zone=default # Generated with "uuidgen". local.storage.uuid= +# Enable TLS for image server transfers. +# When enabled, certificate and key paths must both be configured. +# image.server.tls.enabled=false +# image.server.tls.cert.file=/etc/cloudstack/agent/cloud.crt +# image.server.tls.key.file=/etc/cloudstack/agent/cloud.key + # Location for KVM virtual router scripts. # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/". domr.scripts.dir=scripts/network/domr/kvm diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 3364f9708cf5..22a25eaa6d89 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -123,6 +123,27 @@ public class AgentProperties{ */ public static final Property LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/"); + /** + * Enables TLS on the KVM image server transfer endpoint.
+ * Data type: Boolean.
+ * Default value: false + */ + public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", false); + + /** + * PEM certificate file used by the KVM image server when TLS is enabled.
+ * Data type: String.
+ * Default value: null + */ + public static final Property IMAGE_SERVER_TLS_CERT_FILE = new Property<>("image.server.tls.cert.file", null, String.class); + + /** + * PEM private key file used by the KVM image server when TLS is enabled.
+ * Data type: String.
+ * Default value: null + */ + public static final Property IMAGE_SERVER_TLS_KEY_FILE = new Property<>("image.server.tls.key.file", null, String.class); + /** * Directory where Qemu sockets are placed.
* These sockets are for the Qemu Guest Agent and SSVM provisioning.
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 34f166a7fb69..675c9cde2667 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -398,6 +398,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String vmActivityCheckPath; private String nasBackupPath; private String imageServerPath; + private boolean imageServerTlsEnabled = false; + private String imageServerTlsCertFile; + private String imageServerTlsKeyFile; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -816,6 +819,18 @@ public String getImageServerPath() { return imageServerPath; } + public boolean isImageServerTlsEnabled() { + return imageServerTlsEnabled; + } + + public String getImageServerTlsCertFile() { + return imageServerTlsCertFile; + } + + public String getImageServerTlsKeyFile() { + return imageServerTlsKeyFile; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -1034,6 +1049,14 @@ public boolean configure(final String name, final Map params) th cachePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_CACHE_LOCATION); + imageServerTlsEnabled = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_ENABLED); + imageServerTlsCertFile = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_CERT_FILE); + imageServerTlsKeyFile = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_KEY_FILE); + + if (imageServerTlsEnabled && (StringUtils.isBlank(imageServerTlsCertFile) || StringUtils.isBlank(imageServerTlsKeyFile))) { + throw new ConfigurationException("image server TLS is enabled but image.server.tls.cert.file or image.server.tls.key.file is missing"); + } + params.put("domr.scripts.dir", domrScriptsDir); virtRouterResource = new VirtualRoutingResource(this); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 859b7498859c..1b9b33f83a9b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -40,10 +40,23 @@ public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputingResource resource) { final String imageServerPackageDir = resource.getImageServerPath(); final String imageServerParentDir = new File(imageServerPackageDir).getParent(); final String imageServerModuleName = new File(imageServerPackageDir).getName(); + final String listenAddress = "0.0.0.0"; + final boolean tlsEnabled = resource.isImageServerTlsEnabled(); String unitName = "cloudstack-image-server"; Script checkScript = new Script("/bin/bash", logger); @@ -54,14 +67,21 @@ private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputi return true; } + resetService(unitName); if (checkResult != null) { - String systemdRunCmd = String.format( - "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen 0.0.0.0 --port %d", - unitName, imageServerParentDir, imageServerModuleName, imageServerPort); + StringBuilder systemdRunCmd = new StringBuilder(String.format( + "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen %s --port %d", + unitName, shellQuote(imageServerParentDir), imageServerModuleName, shellQuote(listenAddress), imageServerPort)); + + if (tlsEnabled) { + systemdRunCmd.append(" --tls-enabled"); + systemdRunCmd.append(" --tls-cert-file ").append(shellQuote(resource.getImageServerTlsCertFile())); + systemdRunCmd.append(" --tls-key-file ").append(shellQuote(resource.getImageServerTlsKeyFile())); + } Script startScript = new Script("/bin/bash", logger); startScript.add("-c"); - startScript.add(systemdRunCmd); + startScript.add(systemdRunCmd.toString()); String startResult = startScript.execute(); if (startResult != null) { @@ -144,7 +164,8 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r return new CreateImageTransferAnswer(cmd, false, "Failed to register transfer with image server."); } - final String transferUrl = String.format("http://%s:%d/images/%s", resource.getPrivateIp(), imageServerPort, transferId); + final String transferScheme = resource.isImageServerTlsEnabled() ? "https" : "http"; + final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, resource.getPrivateIp(), imageServerPort, transferId); return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java index 240dc3622192..6c95720fda7d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -39,9 +39,8 @@ private void resetService(String unitName) { resetScript.execute(); } - private boolean stopImageServer() { + private boolean stopImageServer(int imageServerPort) { String unitName = "cloudstack-image-server"; - final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); @@ -81,6 +80,7 @@ private void removeFirewallRule(int port) { public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { final String transferId = cmd.getTransferId(); + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; if (StringUtils.isBlank(transferId)) { return new Answer(cmd, false, "transferId is empty."); } @@ -88,12 +88,12 @@ public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); if (activeTransfers < 0) { logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); - stopImageServer(); + stopImageServer(imageServerPort); return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); } if (activeTransfers == 0) { - stopImageServer(); + stopImageServer(imageServerPort); } return new Answer(cmd, true, "Image transfer finalized."); diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index 6d6648030d1d..99318bc58fc4 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -20,6 +20,7 @@ import logging import os import socket +import ssl import threading from http.server import HTTPServer from socketserver import ThreadingMixIn @@ -184,8 +185,26 @@ def main() -> None: default=CONTROL_SOCKET, help="Path to the Unix domain control socket", ) + parser.add_argument( + "--tls-enabled", + action="store_true", + help="Enable TLS for the HTTP transfer endpoint", + ) + parser.add_argument( + "--tls-cert-file", + default=None, + help="Path to PEM certificate file used when TLS is enabled", + ) + parser.add_argument( + "--tls-key-file", + default=None, + help="Path to PEM private key file used when TLS is enabled", + ) args = parser.parse_args() + if args.tls_enabled and (not args.tls_cert_file or not args.tls_key_file): + parser.error("--tls-enabled requires --tls-cert-file and --tls-key-file") + logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", @@ -204,5 +223,23 @@ def main() -> None: addr = (args.listen, args.port) httpd = ThreadingHTTPServer(addr, handler_cls) - logging.info("listening on http://%s:%d", args.listen, args.port) + + scheme = "http" + if args.tls_enabled: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + if hasattr(ssl, "TLSVersion") and hasattr(context, "minimum_version"): + context.minimum_version = ssl.TLSVersion.TLSv1_2 + else: + if hasattr(ssl, "OP_NO_TLSv1"): + context.options |= ssl.OP_NO_TLSv1 + if hasattr(ssl, "OP_NO_TLSv1_1"): + context.options |= ssl.OP_NO_TLSv1_1 + + context.load_cert_chain(certfile=args.tls_cert_file, keyfile=args.tls_key_file) + + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + scheme = "https" + + logging.info("listening on %s://%s:%d", scheme, args.listen, args.port) httpd.serve_forever() From 260e6bc5bf9f53c0c18be61ffaf8f0720815c274 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Apr 2026 10:07:22 +0530 Subject: [PATCH 084/129] storage pool type fix Signed-off-by: Abhishek Kumar --- .../veeam/api/converter/StoreVOToStorageDomainConverter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java index dcfdcb67a56a..a70eceb1b462 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java @@ -218,9 +218,9 @@ private static String mapPrimaryStorageType(StoragePoolJoinVO pool) { Object t = pool.getPoolType(); // often StoragePoolType enum if (t != null) { String s = t.toString().toLowerCase(); - if (s.contains("networkfilesystem") || s.contains("nfs")) return "nfs"; + if (s.contains("networkfilesystem") || s.contains("nfs") || s.contains("sharedmountpoint")) return "nfs"; if (s.contains("iscsi")) return "iscsi"; - if (s.contains("filesystem")) return "posixfs"; + if (s.contains("filesystem")) return "localfs"; if (s.contains("rbd") || s.contains("ceph")) return "cinder"; // not perfect; pick stable } } catch (Exception ignored) { } From bad164c991c6a08bc90b0b1ec39adb2e7890dbfc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Apr 2026 11:16:15 +0530 Subject: [PATCH 085/129] fixes Signed-off-by: Abhishek Kumar --- .../api/command/user/vm/BaseDeployVMCmd.java | 4 +- .../api/command/user/vm/DeployVMCmd.java | 8 + .../java/com/cloud/dc/dao/ClusterDao.java | 3 +- .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 5 +- .../com/cloud/network/dao/NetworkDao.java | 5 +- .../com/cloud/network/dao/NetworkDaoImpl.java | 13 +- .../com/cloud/tags/dao/ResourceTagDao.java | 3 +- .../cloud/tags/dao/ResourceTagsDaoImpl.java | 5 +- .../apache/cloudstack/veeam/RouteHandler.java | 9 +- .../veeam/adapter/ServerAdapter.java | 379 ++++++++++++------ .../cloudstack/veeam/api/ApiService.java | 4 - .../veeam/api/ClustersRouteHandler.java | 15 +- .../veeam/api/DataCentersRouteHandler.java | 31 +- .../veeam/api/DisksRouteHandler.java | 4 +- .../veeam/api/HostsRouteHandler.java | 15 +- .../veeam/api/ImageTransfersRouteHandler.java | 4 +- .../veeam/api/JobsRouteHandler.java | 2 +- .../veeam/api/NetworksRouteHandler.java | 4 +- .../veeam/api/TagsRouteHandler.java | 4 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 70 +--- .../veeam/api/VnicProfilesRouteHandler.java | 4 +- .../converter/HostJoinVOToHostConverter.java | 2 +- .../converter/UserVmJoinVOToVmConverter.java | 8 +- .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 54 ++- .../apache/cloudstack/veeam/api/dto/Vm.java | 31 ++ .../veeam/api/request/ListQuery.java | 141 +++++++ .../com/cloud/api/query/dao/HostJoinDao.java | 3 +- .../cloud/api/query/dao/HostJoinDaoImpl.java | 5 +- .../api/query/dao/StoragePoolJoinDao.java | 3 + .../api/query/dao/StoragePoolJoinDaoImpl.java | 10 + .../cloud/api/query/dao/UserVmJoinDao.java | 3 +- .../api/query/dao/UserVmJoinDaoImpl.java | 7 +- .../cloud/api/query/dao/VolumeJoinDao.java | 3 +- .../api/query/dao/VolumeJoinDaoImpl.java | 5 +- .../com/cloud/api/query/vo/UserVmJoinVO.java | 6 +- .../backup/KVMBackupExportServiceImpl.java | 8 + .../com/cloud/vpc/dao/MockNetworkDaoImpl.java | 8 +- 37 files changed, 630 insertions(+), 258 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 28e9052124ed..0fffefaee3fc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -150,7 +150,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme protected String userData; @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") - private Long userdataId; + protected Long userdataId; @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") private Map userdataDetails; @@ -200,7 +200,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." + " Mutually exclusive with affinitygroupnames parameter") - private List affinityGroupIdList; + protected List affinityGroupIdList; @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index f94012861929..13baf0fe4ccc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -155,6 +155,14 @@ public void setDisplayVm(Boolean displayVm) { this.displayVm = displayVm; } + public void setUserDataId(Long userDataId) { + this.userdataId = userDataId; + } + + public void setAffinityGroupIds(List ids) { + this.affinityGroupIdList = ids; + } + public void setDetails(Map details) { this.details = details; } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 7952147490ee..76509d2a6d1e 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -23,6 +23,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -62,5 +63,5 @@ public interface ClusterDao extends GenericDao { List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, HypervisorType hypervisorType, CPU.CPUArch arch); - List listByHypervisorType(HypervisorType hypervisorType); + List listByHypervisorType(HypervisorType hypervisorType, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index 8988522fc963..1e36e0a780dd 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -38,6 +38,7 @@ import com.cloud.org.Grouping; import com.cloud.org.Managed; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -415,9 +416,9 @@ public List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, Hypervi } @Override - public List listByHypervisorType(HypervisorType hypervisorType) { + public List listByHypervisorType(HypervisorType hypervisorType, Filter filter) { SearchCriteria sc = ZoneHyTypeSearch.create(); sc.setParameters("hypervisorType", hypervisorType.toString()); - return listBy(sc); + return listBy(sc, filter); } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index 341f9d7cb848..243a9906486e 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -24,6 +24,7 @@ import com.cloud.network.Network.GuestType; import com.cloud.network.Network.State; import com.cloud.network.Networks.TrafficType; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.fsm.StateDao; @@ -96,9 +97,11 @@ public interface NetworkDao extends GenericDao, StateDao serviceProviderMap); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType, Filter filter); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType); - List listByTrafficType(TrafficType trafficType); + List listByTrafficType(TrafficType trafficType, Filter filter); void setCheckForGc(long networkId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 9a01a8ee7e3b..218c447e3bc0 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -632,20 +632,25 @@ public List listBy(final long accountId, final long dataCenterId, fin } @Override - public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType, Filter filter) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("datacenter", zoneId); sc.setParameters("trafficType", trafficType); - return listBy(sc, null); + return listBy(sc, filter); } @Override - public List listByTrafficType(final TrafficType trafficType) { + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + return listByZoneAndTrafficType(zoneId, trafficType, null); + } + + @Override + public List listByTrafficType(final TrafficType trafficType, Filter filter) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("trafficType", trafficType); - return listBy(sc, null); + return listBy(sc, filter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index 5f2225c410f5..ccb6fea2059c 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -23,6 +23,7 @@ import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import org.apache.cloudstack.api.response.ResourceTagResponse; @@ -61,5 +62,5 @@ public interface ResourceTagDao extends GenericDao { List listByResourceUuid(String resourceUuid); - List listByResourceType(ResourceObjectType resourceType); + List listByResourceType(ResourceObjectType resourceType, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index 6fb7f71b269d..091078f46289 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -28,6 +28,7 @@ import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -122,9 +123,9 @@ public List listByResourceUuid(String resourceUuid) { } @Override - public List listByResourceType(ResourceObjectType resourceType) { + public List listByResourceType(ResourceObjectType resourceType, Filter filter) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("resourceType", resourceType); - return listBy(sc); + return listBy(sc, filter); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index 4e0381be6992..d59ef9e2f795 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -19,7 +19,7 @@ import java.io.BufferedReader; import java.io.IOException; -import java.util.Map; +import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -30,6 +30,7 @@ import com.cloud.utils.component.Adapter; public interface RouteHandler extends Adapter { + static final Pattern PAGE_PATTERN = Pattern.compile("\\bpage\\s+(\\d+)"); default int priority() { return 0; } boolean canHandle(String method, String path) throws IOException; void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) @@ -73,10 +74,4 @@ static String getRequestData(HttpServletRequest req) { return null; } } - - static Map getRequestParams(HttpServletRequest req) { - return req.getParameterMap().entrySet().stream() - .filter(e -> e.getValue() != null && e.getValue().length > 0) - .collect(java.util.stream.Collectors.toMap(Map.Entry::getKey, e -> e.getValue()[0])); - } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index a0eed5dbfc14..ae5eb6e0717d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -36,6 +36,9 @@ import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.Rule; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; @@ -116,20 +119,19 @@ import org.apache.cloudstack.veeam.api.dto.VmAction; import org.apache.cloudstack.veeam.api.dto.VnicProfile; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import com.cloud.api.query.dao.AsyncJobJoinDao; import com.cloud.api.query.dao.DataCenterJoinDao; import com.cloud.api.query.dao.HostJoinDao; -import com.cloud.api.query.dao.ImageStoreJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.dao.VolumeJoinDao; import com.cloud.api.query.vo.AsyncJobJoinVO; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.api.query.vo.HostJoinVO; -import com.cloud.api.query.vo.ImageStoreJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.api.query.vo.VolumeJoinVO; @@ -152,6 +154,7 @@ import com.cloud.projects.Project; import com.cloud.projects.ProjectService; import com.cloud.server.ResourceTag; +import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; @@ -166,15 +169,18 @@ import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.user.UserAccount; +import com.cloud.user.UserDataVO; +import com.cloud.user.dao.UserDataDao; import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Filter; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.NicVO; -import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.NicDao; @@ -183,8 +189,7 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -// ToDo: fix list APIs to support pagination, etc -// ToDo: check access on objects +// ToDo: check access for list APIs when not ROOT admin public class ServerAdapter extends ManagerBase { private static final String SERVICE_ACCOUNT_NAME = "veemserviceuser"; @@ -206,7 +211,7 @@ public class ServerAdapter extends ManagerBase { ResizeVolumeCmd.class, ListNetworksCmd.class ); - public static final String GUEST_CPU_MODE = "host-passthrough"; + public static final String WORKER_VM_GUEST_CPU_MODE = "host-passthrough"; @Inject RoleService roleService; @@ -223,9 +228,6 @@ public class ServerAdapter extends ManagerBase { @Inject StoragePoolJoinDao storagePoolJoinDao; - @Inject - ImageStoreJoinDao imageStoreJoinDao; - @Inject ClusterDao clusterDao; @@ -275,7 +277,7 @@ public class ServerAdapter extends ManagerBase { VMTemplateDao templateDao; @Inject - UserVmService userVmService; + UserVmManager userVmManager; @Inject NicDao nicDao; @@ -304,6 +306,12 @@ public class ServerAdapter extends ManagerBase { @Inject ProjectService projectService; + @Inject + AffinityGroupDao affinityGroupDao; + + @Inject + UserDataDao userDataDao; + protected static Tag getDummyTagByName(String name) { Tag tag = new Tag(); String id = UUID.nameUUIDFromBytes(String.format("veeam:%s", name.toLowerCase()).getBytes()).toString(); @@ -429,15 +437,22 @@ protected void waitForJobCompletion(AsyncJobJoinVO job) { waitForJobCompletion(job.getId()); } + protected void validateServiceAccountAdminAccess() { + Pair serviceAccount = getServiceAccount(); + if (!accountService.isAdmin(serviceAccount.second().getId())) { + throw new InvalidParameterValueException("Service account does not have access"); + } + } + @Override public boolean start() { getServiceAccount(); - //find public custom disk offering return true; } - public List listAllDataCenters() { - final List clusters = dataCenterJoinDao.listAll(); + public List listAllDataCenters(Long offset, Long limit) { + Filter filter = new Filter(DataCenterJoinVO.class, "id", true, offset, limit); + final List clusters = dataCenterJoinDao.listAll(filter); return DataCenterJoinVOToDataCenterConverter.toDCList(clusters); } @@ -449,81 +464,92 @@ public DataCenter getDataCenter(String uuid) { return DataCenterJoinVOToDataCenterConverter.toDataCenter(vo); } - public List listStorageDomainsByDcId(final String uuid) { - final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); + public List listStorageDomainsByDcId(final String uuid, final Long offset, final Long limit) { + final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(uuid); if (dataCenterVO == null) { throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); } - List storagePoolVOS = storagePoolJoinDao.listAll(); - List storageDomains = StoreVOToStorageDomainConverter.toStorageDomainListFromPools(storagePoolVOS); - List imageStoreJoinVOS = imageStoreJoinDao.listAll(); - storageDomains.addAll(StoreVOToStorageDomainConverter.toStorageDomainListFromStores(imageStoreJoinVOS)); - return storageDomains; + validateServiceAccountAdminAccess(); + Filter filter = new Filter(StoragePoolJoinVO.class, "id", true, offset, limit); + List storagePoolVOS = storagePoolJoinDao.listByZoneAndProvider(dataCenterVO.getId(), filter); + return StoreVOToStorageDomainConverter.toStorageDomainListFromPools(storagePoolVOS); } - public List listNetworksByDcId(final String uuid) { + public List listNetworksByDcId(final String uuid, final Long offset, final Long limit) { final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); if (dataCenterVO == null) { throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); } - List networks = networkDao.listByZoneAndTrafficType(dataCenterVO.getId(), Networks.TrafficType.Guest); + validateServiceAccountAdminAccess(); + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + List networks = networkDao.listByZoneAndTrafficType(dataCenterVO.getId(), Networks.TrafficType.Guest, filter); return NetworkVOToNetworkConverter.toNetworkList(networks, (dcId) -> dataCenterVO); } - public List listAllClusters() { - final List clusters = clusterDao.listByHypervisorType(Hypervisor.HypervisorType.KVM); + public List listAllClusters(Long offset, Long limit) { + validateServiceAccountAdminAccess(); + Filter filter = new Filter(ClusterVO.class, "id", true, offset, limit); + final List clusters = clusterDao.listByHypervisorType(Hypervisor.HypervisorType.KVM, filter); return ClusterVOToClusterConverter.toClusterList(clusters, this::getZoneById); } public Cluster getCluster(String uuid) { + validateServiceAccountAdminAccess(); final ClusterVO vo = clusterDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); } - return ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); + return ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); } - public List listAllHosts() { - final List hosts = hostJoinDao.listRoutingHostsByHypervisor(Hypervisor.HypervisorType.KVM); + public List listAllHosts(Long offset, Long limit) { + validateServiceAccountAdminAccess(); + Filter filter = new Filter(HostJoinVO.class, "id", true, offset, limit); + final List hosts = hostJoinDao.listRoutingHostsByHypervisor(Hypervisor.HypervisorType.KVM, filter); return HostJoinVOToHostConverter.toHostList(hosts); } public Host getHost(String uuid) { + validateServiceAccountAdminAccess(); final HostJoinVO vo = hostJoinDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); } - return HostJoinVOToHostConverter.toHost(vo); + return HostJoinVOToHostConverter.toHost(vo); } - public List listAllNetworks() { - final List networks = networkDao.listAll(); + public List listAllNetworks(Long offset, Long limit) { + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest, filter); return NetworkVOToNetworkConverter.toNetworkList(networks, this::getZoneById); } public Network getNetwork(String uuid) { final NetworkVO vo = networkDao.findByUuid(uuid); if (vo == null) { - throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + throw new InvalidParameterValueException("Network with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); } - public List listAllVnicProfiles() { - final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest); + public List listAllVnicProfiles(Long offset, Long limit) { + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest, filter); return NetworkVOToVnicProfileConverter.toVnicProfileList(networks, this::getZoneById); } public VnicProfile getVnicProfile(String uuid) { final NetworkVO vo = networkDao.findByUuid(uuid); if (vo == null) { - throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + throw new InvalidParameterValueException("Nic profile with ID " + uuid + " not found"); } return NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); } - public List listAllInstances() { - List vms = userVmJoinDao.listByHypervisorType(Hypervisor.HypervisorType.KVM); + public List listAllInstances(Long offset, Long limit) { + Filter filter = new Filter(UserVmJoinVO.class, "id", true, offset, limit); + List vms = userVmJoinDao.listByHypervisorType(Hypervisor.HypervisorType.KVM, filter); return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); } @@ -539,17 +565,24 @@ public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, bo allContent); } - Ternary getVmOwner(Vm request) { + Account getOwnerForInstanceCreation(Vm request) { if (!VeeamControlService.InstanceRestoreAssignOwner.value()) { - return new Ternary<>(null, null, null); + return null; } String accountUuid = request.getAccountId(); if (StringUtils.isBlank(accountUuid)) { - return new Ternary<>(null, null, null); + return null; } Account account = accountService.getActiveAccountByUuid(accountUuid); if (account == null) { logger.warn("Account with ID {} not found, unable to determine owner for VM creation request", accountUuid); + return null; + } + return account; + } + + Ternary getOwnerDetailsForInstanceCreation(Account account) { + if (account == null) { return new Ternary<>(null, null, null); } String accountName = account.getAccountName(); @@ -576,7 +609,7 @@ public Vm createInstance(Vm request) { throw new InvalidParameterValueException("Invalid name specified for the VM"); } String displayName = name; - name = name.replaceAll("_", "-"); + name = name.replace("_", "-"); Long zoneId = null; Long clusterId = null; if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { @@ -589,6 +622,10 @@ public Vm createInstance(Vm request) { if (zoneId == null) { throw new InvalidParameterValueException("Failed to determine datacenter for VM creation request"); } + DataCenterVO zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("DataCenter could not be determined for the request"); + } Integer cpu = null; try { cpu = Integer.valueOf(request.getCpu().getTopology().getSockets()); @@ -605,12 +642,14 @@ public Vm createInstance(Vm request) { if (memory == null) { throw new InvalidParameterValueException("Memory must be specified"); } + int memoryMB = (int)(memory / (1024L * 1024L)); String userdata = null; if (request.getInitialization() != null) { userdata = request.getInitialization().getCustomScript(); } Pair bootOptions = Vm.Bios.retrieveBootOptions(request.getBios()); - Ternary owner = getVmOwner(request); + Account owner = getOwnerForInstanceCreation(request); + Ternary ownerDetails = getOwnerDetailsForInstanceCreation(owner); String serviceOfferingUuid = null; if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { serviceOfferingUuid = request.getCpuProfile().getId(); @@ -620,29 +659,68 @@ public Vm createInstance(Vm request) { templateUuid = request.getTemplate().getId(); } Pair serviceUserAccount = getServiceAccount(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { - return createInstance(zoneId, clusterId, owner.first(), owner.second(), owner.third(), name, displayName, - serviceOfferingUuid, cpu, memory, templateUuid, userdata, bootOptions.first(), bootOptions.second()); + return createInstance(zone, clusterId, owner, ownerDetails.first(), ownerDetails.second(), + ownerDetails.third(), name, displayName, serviceOfferingUuid, cpu, memoryMB, templateUuid, + userdata, bootOptions.first(), bootOptions.second(), request.getAffinityGroupId(), + request.getUserDataId(), request.getDetails()); } finally { CallContext.unregister(); } } - protected ServiceOffering getServiceOfferingIdForVmCreation(String serviceOfferingUuid, long zoneId, int cpu, long memory) { - if (StringUtils.isNotBlank(serviceOfferingUuid)) { - ServiceOffering offering = serviceOfferingDao.findByUuid(serviceOfferingUuid); - if (offering != null && !offering.isCustomized()) { - // ToDo: check offering is available in the specified zone and matches the requested cpu/memory if it's not a custom offering - return offering; + protected ServiceOffering getServiceOfferingFromRequest(com.cloud.dc.DataCenter zone, Account account, + String uuid, int cpu, int memory) { + if (StringUtils.isBlank(uuid)) { + return null; + } + ServiceOfferingVO offering = serviceOfferingDao.findByUuid(uuid); + if (offering == null) { + logger.warn("Service offering with ID {} linked with the VM request not found", uuid); + return null; + } + try { + accountService.checkAccess(account, offering, zone); + } catch (PermissionDeniedException e) { + logger.warn("Service offering with ID {} linked with the VM request is not accessible for the account {}. Offering: {}, zone: {}", + uuid, account, offering, zone); + return null; + } + if (!offering.isCustomized() && (offering.getCpu() != cpu || offering.getRamSize() != memory)) { + logger.warn("Service offering with ID {} linked with the VM request has different CPU or memory than requested. Offering: {}, requested CPU: {}, requested memory: {}", + uuid, offering, cpu, memory); + return null; + } + if (offering.isCustomized()) { + Map params = Map.of( + VmDetailConstants.CPU_NUMBER, String.valueOf(cpu), + VmDetailConstants.MEMORY, String.valueOf(memory) + ); + try { + userVmManager.validateCustomParameters(offering, params); + offering.setCpu(cpu); + offering.setRamSize(memory); + } catch (InvalidParameterValueException e) { + logger.warn("Service offering with ID {} linked with the VM request is customized but does not support requested CPU or memory. Offering: {}, requested CPU: {}, requested memory: {}", + uuid, offering, cpu, memory); + return null; } } + return offering; + } + + protected ServiceOffering getServiceOfferingIdForVmCreation(com.cloud.dc.DataCenter zone, Account account, + String serviceOfferingUuid, int cpu, int memory) { + ServiceOffering offering = getServiceOfferingFromRequest(zone, account, serviceOfferingUuid, cpu, memory); + if (offering != null) { + return offering; + } ListServiceOfferingsCmd cmd = new ListServiceOfferingsCmd(); ComponentContext.inject(cmd); - cmd.setZoneId(zoneId); + cmd.setZoneId(zone.getId()); cmd.setCpuNumber(cpu); - Integer memoryMB = (int)(memory / (1024L * 1024L)); - cmd.setMemory(memoryMB); + cmd.setMemory(memory); ListResponse offerings = queryService.searchForServiceOfferings(cmd); if (offerings.getResponses().isEmpty()) { return null; @@ -651,7 +729,7 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(String serviceOfferi return serviceOfferingDao.findByUuid(uuid); } - protected VMTemplateVO getTemplateForVmCreation(String templateUuid) { + protected VMTemplateVO getTemplateForInstanceCreation(String templateUuid) { if (StringUtils.isBlank(templateUuid)) { return null; } @@ -663,17 +741,20 @@ protected VMTemplateVO getTemplateForVmCreation(String templateUuid) { return template; } - protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String accountName, Long projectId, - String name, String displayName, String serviceOfferingUuid, int cpu, long memory, String templateUuid, - String userdata, ApiConstants.BootType bootType, ApiConstants.BootMode bootMode) { - ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(serviceOfferingUuid, zoneId, cpu, memory); + protected Vm createInstance(com.cloud.dc.DataCenter zone, Long clusterId, Account owner, Long domainId, + String accountName, Long projectId, String name, String displayName, String serviceOfferingUuid, + int cpu, int memory, String templateUuid, String userdata, ApiConstants.BootType bootType, + ApiConstants.BootMode bootMode, String affinityGroupId, String userDataId, Map details) { + Account account = owner != null ? owner : CallContext.current().getCallingAccount(); + ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zone, account, serviceOfferingUuid, cpu, + memory); if (serviceOffering == null) { throw new CloudRuntimeException("No service offering found for VM creation with specified CPU and memory"); } DeployVMCmdByAdmin cmd = new DeployVMCmdByAdmin(); cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); - cmd.setZoneId(zoneId); + cmd.setZoneId(zone.getId()); cmd.setClusterId(clusterId); if (domainId != null && StringUtils.isNotEmpty(accountName)) { cmd.setDomainId(domainId); @@ -696,22 +777,39 @@ protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String a if (bootMode != null) { cmd.setBootMode(bootMode.toString()); } - VMTemplateVO template = getTemplateForVmCreation(templateUuid); + VMTemplateVO template = getTemplateForInstanceCreation(templateUuid); if (template != null) { cmd.setTemplateId(template.getId()); } - // ToDo: handle any other field? - // Handle custom offerings + if (StringUtils.isNotBlank(affinityGroupId)) { + AffinityGroupVO group = affinityGroupDao.findByUuid(affinityGroupId); + if (group == null) { + logger.warn("Failed to find affinity group with ID {} specified in Instance creation request, " + + "skipping affinity group assignment", affinityGroupId); + } else { + cmd.setAffinityGroupIds(List.of(group.getId())); + } + } + if (StringUtils.isNotBlank(userDataId)) { + UserDataVO userData = userDataDao.findByUuid(userDataId); + if (userData == null) { + logger.warn("Failed to find userdata with ID {} specified in Instance creation request, " + + "skipping userdata assignment", userDataId); + } else { + cmd.setUserDataId(userData.getId()); + } + } cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); + Map instanceDetails = getDetailsForInstanceCreation(userdata, serviceOffering, details); + if (MapUtils.isNotEmpty(instanceDetails)) { + Map> map = new HashMap<>(); + map.put(0, details); + cmd.setDetails(map); + } cmd.setBlankInstance(true); - Map details = new HashMap<>(); - details.put(VmDetailConstants.GUEST_CPU_MODE, GUEST_CPU_MODE); - Map> map = new HashMap<>(); - map.put(0, details); - cmd.setDetails(map); try { - UserVm vm = userVmService.createVirtualMachine(cmd); - vm = userVmService.finalizeCreateVirtualMachine(vm.getId()); + UserVm vm = userVmManager.createVirtualMachine(cmd); + vm = userVmManager.finalizeCreateVirtualMachine(vm.getId()); UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); @@ -720,6 +818,35 @@ protected Vm createInstance(Long zoneId, Long clusterId, Long domainId, String a } } + @NotNull + private static Map getDetailsForInstanceCreation(String userdata, ServiceOffering serviceOffering, + Map existingDetails) { + Map details = new HashMap<>(); + List detailsTobeSkipped = List.of( + ApiConstants.BootType.BIOS.toString(), + ApiConstants.BootType.UEFI.toString()); + if (MapUtils.isNotEmpty(existingDetails)) { + for (Map.Entry entry : existingDetails.entrySet()) { + if (detailsTobeSkipped.contains(entry.getKey())) { + continue; + } + details.put(entry.getKey(), entry.getValue()); + } + } + if (StringUtils.isNotEmpty(userdata)) { + // Assumption: Only worker VM will have userdata and it needs CPU mode + details.put(VmDetailConstants.GUEST_CPU_MODE, WORKER_VM_GUEST_CPU_MODE); + } + if (serviceOffering.isCustomized()) { + details.put(VmDetailConstants.CPU_NUMBER, String.valueOf(serviceOffering.getCpu())); + details.put(VmDetailConstants.MEMORY, String.valueOf(serviceOffering.getRamSize())); + if (serviceOffering.getSpeed() == null && !details.containsKey(VmDetailConstants.CPU_SPEED)) { + details.put(VmDetailConstants.CPU_SPEED, String.valueOf(1000)); + } + } + return details; + } + public Vm updateInstance(String uuid, Vm request) { logger.warn("Received request to update VM with ID {}. No action, returning existing VM data.", uuid); return getInstance(uuid, false, false, false); @@ -856,51 +983,27 @@ protected Long getVolumePhysicalSize(VolumeJoinVO vo) { return volumeApiService.getVolumePhysicalSize(vo.getFormat(), vo.getPath(), vo.getChainInfo()); } - public List listAllDisks() { - List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM); + public List listAllDisks(Long offset, Long limit) { + Filter filter = new Filter(VolumeJoinVO.class, "id", true, offset, limit); + List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM, filter); return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes, this::getVolumePhysicalSize); } public Disk getDisk(String uuid) { - VolumeJoinVO vo = volumeJoinDao.findByUuid(uuid); + VolumeVO vo = volumeDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); } - return VolumeJoinVOToDiskConverter.toDisk(vo, this::getVolumePhysicalSize); + accountService.checkAccess(getServiceAccount().second(), null, false, vo); + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findByUuid(uuid), this::getVolumePhysicalSize); } public Disk copyDisk(String uuid) { throw new InvalidParameterValueException("Copy Disk with ID " + uuid + " not implemented"); -// VolumeVO vo = volumeDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); -// } -// Pair serviceUserAccount = createServiceAccountIfNeeded(); -// CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); -// try { -// Volume volume = volumeApiService.copyVolume(vo.getId(), vo.getName() + "_copy", null, null); -// VolumeJoinVO copiedVolumeVO = volumeJoinDao.findById(volume.getId()); -// return VolumeJoinVOToDiskConverter.toDisk(copiedVolumeVO); -// } finally { -// CallContext.unregister(); -// } } public Disk reduceDisk(String uuid) { throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); -// VolumeVO vo = volumeDao.findByUuid(uuid); -// if (vo == null) { -// throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); -// } -// Pair serviceUserAccount = createServiceAccountIfNeeded(); -// CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); -// try { -// Volume volume = volumeApiService.reduceDisk(vo.getId(), vo.getName() + "_copy", null, null); -// VolumeJoinVO copiedVolumeVO = volumeJoinDao.findById(volume.getId()); -// return VolumeJoinVOToDiskConverter.toDisk(copiedVolumeVO); -// } finally { -// CallContext.unregister(); -// } } protected List listDiskAttachmentsByInstanceId(final long instanceId) { @@ -913,6 +1016,7 @@ public List listDiskAttachmentsByInstanceUuid(final String uuid) if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return listDiskAttachmentsByInstanceId(vo.getId()); } @@ -953,6 +1057,8 @@ public DiskAttachment attachInstanceDisk(final String vmUuid, final DiskAttachme if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } + Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); if (request == null || request.getDisk() == null || StringUtils.isEmpty(request.getDisk().getId())) { throw new InvalidParameterValueException("Request disk data is empty"); } @@ -960,7 +1066,7 @@ public DiskAttachment attachInstanceDisk(final String vmUuid, final DiskAttachme if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } - Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); if (vmVo.getAccountId() != volumeVO.getAccountId()) { if (VeeamControlService.InstanceRestoreAssignOwner.value()) { assignVolumeToAccount(volumeVO, vmVo.getAccountId(), serviceUserAccount); @@ -1013,18 +1119,7 @@ public Disk createDisk(Disk request) { if (StringUtils.isBlank(sizeStr)) { throw new InvalidParameterValueException("Provisioned size must be specified"); } - long provisionedSizeInGb; - try { - provisionedSizeInGb = Long.parseLong(sizeStr); - } catch (NumberFormatException ex) { - throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); - } - if (provisionedSizeInGb <= 0) { - throw new InvalidParameterValueException("Provisioned size must be greater than zero"); - } - // round-up provisionedSizeInGb to the next whole GB - long GB = 1024L * 1024L * 1024L; - provisionedSizeInGb = Math.max(1L, (provisionedSizeInGb + GB - 1) / GB); + long provisionedSizeInGb = getProvisionedSizeInGb(sizeStr); Long initialSize = null; if (StringUtils.isNotBlank(request.getInitialSize())) { try { @@ -1049,6 +1144,22 @@ public Disk createDisk(Disk request) { } } + private static long getProvisionedSizeInGb(String sizeStr) { + long provisionedSizeInGb; + try { + provisionedSizeInGb = Long.parseLong(sizeStr); + } catch (NumberFormatException ex) { + throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); + } + if (provisionedSizeInGb <= 0) { + throw new InvalidParameterValueException("Provisioned size must be greater than zero"); + } + // round-up provisionedSizeInGb to the next whole GB + long GB = 1024L * 1024L * 1024L; + provisionedSizeInGb = Math.max(1L, (provisionedSizeInGb + GB - 1) / GB); + return provisionedSizeInGb; + } + @NotNull private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { Volume volume; @@ -1084,6 +1195,7 @@ public List listNicsByInstanceUuid(final String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return listNicsByInstance(vo.getId(), vo.getUuid()); } @@ -1119,7 +1231,7 @@ protected void assignVmToAccount(UserVmVO vmVO, long accountId, Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); if (request == null || request.getVnicProfile() == null || StringUtils.isEmpty(request.getVnicProfile().getId())) { throw new InvalidParameterValueException("Request nic data is empty"); } @@ -1140,7 +1254,7 @@ public Nic attachInstanceNic(final String vmUuid, final Nic request) { if (networkVO == null) { throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().getId() + " not found"); } - Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, networkVO); if (vmVo.getAccountId() != networkVO.getAccountId() && networkVO.getAccountId() != Account.ACCOUNT_ID_SYSTEM && VeeamControlService.InstanceRestoreAssignOwner.value() && @@ -1156,7 +1270,7 @@ public Nic attachInstanceNic(final String vmUuid, final Nic request) { if (request.getMac() != null && StringUtils.isNotBlank(request.getMac().getAddress())) { cmd.setMacAddress(request.getMac().getAddress()); } - userVmService.addNicToVirtualMachine(cmd); + userVmManager.addNicToVirtualMachine(cmd); NicVO nic = nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkVO.getId(), vmVo.getId()); if (nic == null) { throw new CloudRuntimeException("Failed to attach NIC to VM"); @@ -1167,8 +1281,9 @@ public Nic attachInstanceNic(final String vmUuid, final Nic request) { } } - public List listAllImageTransfers() { - List imageTransfers = imageTransferDao.listAll(); + public List listAllImageTransfers(Long offset, Long limit) { + Filter filter = new Filter(ImageTransferVO.class, "id", true, offset, limit); + List imageTransfers = imageTransferDao.listAll(filter); return ImageTransferVOToImageTransferConverter.toImageTransferList(imageTransfers, this::getHostById, this::getVolumeById); } @@ -1177,6 +1292,7 @@ public ImageTransfer getImageTransfer(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); } @@ -1191,6 +1307,8 @@ public ImageTransfer createImageTransfer(ImageTransfer request) { if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } + Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), null, false, volumeVO); Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); if (direction == null) { throw new InvalidParameterValueException("Invalid or missing direction"); @@ -1204,7 +1322,7 @@ public ImageTransfer createImageTransfer(ImageTransfer request) { } backupId = backupVO.getId(); } - return createImageTransfer(backupId, volumeVO.getId(), direction, format); + return createImageTransfer(backupId, volumeVO.getId(), direction, format, serviceUserAccount); } public boolean cancelImageTransfer(String uuid) { @@ -1212,6 +1330,7 @@ public boolean cancelImageTransfer(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); return kvmBackupExportService.cancelImageTransfer(vo.getId()); } @@ -1220,11 +1339,12 @@ public boolean finalizeImageTransfer(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); return kvmBackupExportService.finalizeImageTransfer(vo.getId()); } - private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { - Pair serviceUserAccount = getServiceAccount(); + private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format, + Pair serviceUserAccount) { CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { org.apache.cloudstack.backup.ImageTransfer imageTransfer = @@ -1268,7 +1388,7 @@ protected Map getDetailsByInstanceId(Long instanceId) { return vmInstanceDetailsDao.listDetailsKeyPairs(instanceId, true); } - public List listAllJobs() { + public List listPendingJobs() { Pair serviceUserAccount = getServiceAccount(); List jobIds = asyncJobDao.listPendingJobIdsForAccount(serviceUserAccount.second().getId()); List jobJoinVOs = asyncJobJoinDao.listByIds(jobIds); @@ -1280,6 +1400,7 @@ public Job getJob(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Job with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return AsyncJobJoinVOToJobConverter.toJob(vo); } @@ -1298,6 +1419,7 @@ public Snapshot createInstanceSnapshot(final String vmUuid, final Snapshot reque throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { CreateVMSnapshotCmd cmd = new CreateVMSnapshotCmd(); @@ -1329,6 +1451,7 @@ public Snapshot getSnapshot(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); UserVmVO vm = userVmDao.findById(vo.getVmId()); return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); } @@ -1340,6 +1463,7 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vo); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); @@ -1372,6 +1496,7 @@ public ResourceAction revertInstanceToSnapshot(String uuid, boolean async) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vo); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { RevertToVMSnapshotCmd cmd = new RevertToVMSnapshotCmd(); @@ -1412,6 +1537,7 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartBackupCmd cmd = new StartBackupCmd(); @@ -1442,6 +1568,7 @@ public Backup getBackup(String uuid) { if (vo == null) { throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); return BackupVOToBackupConverter.toBackup(vo, id -> userVmDao.findById(id), this::getHostById, this::getBackupDisks); } @@ -1461,6 +1588,7 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { throw new InvalidParameterValueException("Backup with ID " + backupUuid + " not found"); } Pair serviceUserAccount = getServiceAccount(); + accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, backup); CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { FinalizeBackupCmd cmd = new FinalizeBackupCmd(); @@ -1495,6 +1623,7 @@ public List listCheckpointsByInstanceUuid(final String uuid) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), null, false, vo); Checkpoint checkpoint = UserVmVOToCheckpointConverter.toCheckpoint(vo); if (checkpoint == null) { return Collections.emptyList(); @@ -1507,6 +1636,7 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { if (vo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } + accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); if (!Objects.equals(vo.getActiveCheckpointId(), checkpointId)) { logger.warn("Checkpoint ID {} does not match active checkpoint for VM {}", checkpointId, vmUuid); return; @@ -1525,9 +1655,11 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { } } - public List listAllTags() { + public List listAllTags(final Long offset, final Long limit) { List tags = new ArrayList<>(getDummyTags().values()); - List vmResourceTags = resourceTagDao.listByResourceType(ResourceTag.ResourceObjectType.UserVm); + Filter filter = new Filter(ResourceTagVO.class, "id", true, offset, limit); + List vmResourceTags = resourceTagDao.listByResourceType(ResourceTag.ResourceObjectType.UserVm, + filter); if (CollectionUtils.isNotEmpty(vmResourceTags)) { tags.addAll(ResourceTagVOToTagConverter.toTags(vmResourceTags)); } @@ -1541,6 +1673,7 @@ public Tag getTag(String uuid) { Tag tag = getDummyTags().get(uuid); if (tag == null) { ResourceTagVO resourceTagVO = resourceTagDao.findByUuid(uuid); + accountService.checkAccess(getServiceAccount().second(), null, false, resourceTagVO); if (resourceTagVO != null) { tag = ResourceTagVOToTagConverter.toTag(resourceTagVO); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java index c9024633680d..d076604515a7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java @@ -76,16 +76,12 @@ private static Api createDummyApi(String basePath) { add(links, basePath + "/clusters?search={query}", "clusters/search"); add(links, basePath + "/datacenters", "datacenters"); add(links, basePath + "/datacenters?search={query}", "datacenters/search"); - add(links, basePath + "/events", "events"); - add(links, basePath + "/events;from={event_id}?search={query}", "events/search"); add(links, basePath + "/hosts", "hosts"); add(links, basePath + "/hosts?search={query}", "hosts/search"); add(links, basePath + "/networks", "networks"); add(links, basePath + "/networks?search={query}", "networks/search"); add(links, basePath + "/storagedomains", "storagedomains"); add(links, basePath + "/storagedomains?search={query}", "storagedomains/search"); - add(links, basePath + "/templates", "templates"); - add(links, basePath + "/templates?search={query}", "templates/search"); add(links, basePath + "/vms", "vms"); add(links, basePath + "/vms?search={query}", "vms/search"); add(links, basePath + "/disks", "disks"); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java index c3ee3ab3cdd7..f4107ff3735e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java @@ -29,11 +29,13 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Cluster; import org.apache.cloudstack.veeam.api.dto.NamedList; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.utils.component.ManagerBase; public class ClustersRouteHandler extends ManagerBase implements RouteHandler { @@ -84,9 +86,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllClusters(); - NamedList response = NamedList.of("cluster", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllClusters(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("cluster", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, @@ -96,6 +103,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index bf8e2885251b..4ff5add7d3d8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -31,11 +31,13 @@ import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Network; import org.apache.cloudstack.veeam.api.dto.StorageDomain; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.utils.component.ManagerBase; public class DataCentersRouteHandler extends ManagerBase implements RouteHandler { @@ -81,11 +83,11 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path } else if (idAndSubPath.size() == 2) { String subPath = idAndSubPath.get(1); if ("storagedomains".equals(subPath)) { - handleGetStorageDomainsByDcId(id, resp, outFormat, io); + handleGetStorageDomainsByDcId(id, req, resp, outFormat, io); return; } if ("networks".equals(subPath)) { - handleGetNetworksByDcId(id, resp, outFormat, io); + handleGetNetworksByDcId(id, req, resp, outFormat, io); return; } } @@ -96,7 +98,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllDataCenters(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllDataCenters(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("data_center", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } @@ -111,25 +114,35 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi } } - protected void handleGetStorageDomainsByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleGetStorageDomainsByDcId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { try { - List storageDomains = serverAdapter.listStorageDomainsByDcId(id); + ListQuery query = ListQuery.fromRequest(req); + List storageDomains = serverAdapter.listStorageDomainsByDcId(id, query.getPage(), + query.getMax()); NamedList response = NamedList.of("storage_domain", storageDomains); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } - protected void handleGetNetworksByDcId(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, - final VeeamControlServlet io) throws IOException { + protected void handleGetNetworksByDcId(final String id, final HttpServletRequest req, + final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) + throws IOException { try { - List networks = serverAdapter.listNetworksByDcId(id); + ListQuery query = ListQuery.fromRequest(req); + List networks = serverAdapter.listNetworksByDcId(id, query.getPage(), + query.getMax()); NamedList response = NamedList.of("network", networks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index f0fc1368d56d..f4cd3b6a3785 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Disk; import org.apache.cloudstack.veeam.api.dto.NamedList; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -120,7 +121,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllDisks(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllDisks(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("disk", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java index 54f19424cf93..931291714c6c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java @@ -29,11 +29,13 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Host; import org.apache.cloudstack.veeam.api.dto.NamedList; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.utils.component.ManagerBase; public class HostsRouteHandler extends ManagerBase implements RouteHandler { @@ -84,9 +86,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllHosts(); - NamedList response = NamedList.of("host", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllHosts(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("host", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, @@ -96,6 +103,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (PermissionDeniedException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 33371bc3c354..00b473eb6a41 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.ImageTransfer; import org.apache.cloudstack.veeam.api.dto.NamedList; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -105,7 +106,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllImageTransfers(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllImageTransfers(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("image_transfer", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index a96c80aefe5b..0cb038127697 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -84,7 +84,7 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllJobs(); + final List result = serverAdapter.listPendingJobs(); NamedList response = NamedList.of("job", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index 5e5d9927e65a..4014dc796fe6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Network; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -84,7 +85,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllNetworks(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllNetworks(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("network", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java index e81709cb2121..b571bcaa2ede 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Tag; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -85,7 +86,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllTags(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllTags(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("tag", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index e911f7636de0..fdf542d64714 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -38,10 +38,7 @@ import org.apache.cloudstack.veeam.api.dto.Snapshot; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.cloudstack.veeam.api.dto.VmAction; -import org.apache.cloudstack.veeam.api.request.VmListQuery; -import org.apache.cloudstack.veeam.api.request.VmSearchExpr; -import org.apache.cloudstack.veeam.api.request.VmSearchFilters; -import org.apache.cloudstack.veeam.api.request.VmSearchParser; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -54,24 +51,10 @@ public class VmsRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/vms"; - private static final int DEFAULT_MAX = 50; - private static final int HARD_CAP_MAX = 1000; - private static final int DEFAULT_PAGE = 1; @Inject ServerAdapter serverAdapter; - private VmSearchParser searchParser; - - @Override - public boolean start() { - - this.searchParser = new VmSearchParser(Set.of( - "id", "name", "status", "cluster", "host", "template" - )); - return true; - } - @Override public int priority() { return 5; @@ -248,59 +231,12 @@ protected static boolean isRequestAsync(HttpServletRequest req) { protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final VmListQuery q = fromRequest(req); - - // Validate max/page early (optional strictness) - if (q.getMax() != null && q.getMax() <= 0) { - io.notFound(resp, "Invalid 'max' (must be > 0)", outFormat); - return; - } - if (q.getPage() != null && q.getPage() <= 0) { - io.notFound(resp, "Invalid 'page' (must be > 0)", outFormat); - return; - } - - final int limit = q.resolvedMax(DEFAULT_MAX, HARD_CAP_MAX); - final int offset = q.offset(DEFAULT_MAX, HARD_CAP_MAX, DEFAULT_PAGE); - - final VmSearchExpr expr; - try { - expr = searchParser.parse(q.getSearch()); - } catch (VmSearchParser.VmSearchParseException e) { - io.notFound(resp, "Invalid search: " + e.getMessage(), outFormat); - return; - } - - final VmSearchFilters filters; - try { - filters = VmSearchFilters.fromAndOnly(expr); // AND-only v1 - } catch (VmSearchParser.VmSearchParseException e) { - io.notFound(resp, "Unsupported search: " + e.getMessage(), outFormat); - return; - } - - final List result = serverAdapter.listAllInstances(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllInstances(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("vm", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } - protected static VmListQuery fromRequest(final HttpServletRequest req) { - final VmListQuery q = new VmListQuery(); - q.setSearch(req.getParameter("search")); - q.setMax(parseIntOrNull(req.getParameter("max"))); - q.setPage(parseIntOrNull(req.getParameter("page"))); - return q; - } - - protected static Integer parseIntOrNull(final String s) { - if (s == null || s.trim().isEmpty()) return null; - try { - return Integer.parseInt(s.trim()); - } catch (NumberFormatException e) { - return Integer.valueOf(-1); // will be rejected by validation above - } - } - protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req, logger); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index 28f6b816d14b..3e8aab2176fd 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.VnicProfile; +import org.apache.cloudstack.veeam.api.request.ListQuery; import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; @@ -84,7 +85,8 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listAllVnicProfiles(); + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllVnicProfiles(query.getOffset(), query.getLimit()); NamedList response = NamedList.of("vnic_profile", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java index d627aa4d63ff..4df1dd91e1cc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java @@ -73,7 +73,7 @@ public static Host toHost(final HostJoinVO vo) { // --- Memory --- h.setMemory(String.valueOf(vo.getTotalMemory())); - h.setMaxSchedulingMemory(String.valueOf(vo.getTotalMemory() - vo.getMemUsedCapacity())); // ToDo: check + h.setMaxSchedulingMemory(String.valueOf(vo.getTotalMemory() - vo.getMemUsedCapacity())); // --- OS / versions (optional placeholders) --- // If you want, you can set conservative defaults to match oVirt shape. diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 44691a0ef492..7f148b8d65b9 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -129,8 +129,9 @@ public static Vm toVm(final UserVmJoinVO src, final Function h os.setBoot(boot); dst.setOs(os); Vm.Bios bios = Vm.Bios.getDefault(); + Map details = null; if (detailsResolver != null) { - Map details = detailsResolver.apply(src.getId()); + details = detailsResolver.apply(src.getId()); Vm.Bios.updateBios(bios, MapUtils.getString(details, ApiConstants.BootType.UEFI.toString())); } dst.setBios(bios); @@ -167,6 +168,11 @@ public static Vm toVm(final UserVmJoinVO src, final Function h dst.setInitialization(getOvfInitialization(dst, src)); } + dst.setAccountId(src.getAccountUuid()); + dst.setAffinityGroupId(src.getAffinityGroupUuid()); + dst.setUserDataId(src.getUserDataUuid()); + dst.setDetails(details); + return dst; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java index fcccf299f27f..d417ffde17de 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java @@ -21,8 +21,10 @@ import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.TimeZone; import java.util.UUID; @@ -36,6 +38,7 @@ import javax.xml.xpath.XPathFactory; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Document; @@ -195,6 +198,22 @@ public static String toXml(final Vm vm, final UserVmJoinVO vo) { sb.append(""); } sb.append(""); + if (MapUtils.isNotEmpty(vm.getDetails())) { + sb.append("
"); + for (Map.Entry entry : vm.getDetails().entrySet()) { + sb.append(""); + sb.append("").append(escapeText(entry.getKey())).append(""); + sb.append("").append(escapeText(entry.getValue())).append(""); + sb.append(""); + } + sb.append("
"); + } + if (vo.getUserDataId() != null) { + sb.append("").append(escapeText(vo.getUserDataUuid())).append(""); + } + if (vo.getAffinityGroupId() != null) { + sb.append("").append(escapeText(vo.getAffinityGroupUuid())).append(""); + } sb.append(""); sb.append("
"); } @@ -518,14 +537,35 @@ private static void updateFromXmlCloudStackMetadataSection(Vm vm, Node metadataS if (StringUtils.isNotBlank(serviceOfferingId)) { vm.setCpuProfile(Ref.of("", serviceOfferingId)); } - } - - private static String xpathString(XPath xpath, Document doc, String expression) { + String affinityGroupId = xpathString(xpath, metadataSection, ".//*[local-name()='AffinityGroupId']/text()"); + if (StringUtils.isNotBlank(affinityGroupId)) { + vm.setAffinityGroupId(affinityGroupId); + } + String userDataId = xpathString(xpath, metadataSection, ".//*[local-name()='UserDataId']/text()"); + if (StringUtils.isNotBlank(userDataId)) { + vm.setUserDataId(userDataId); + } + final Map details = new HashMap<>(); try { - String value = (String) xpath.evaluate(expression, doc, XPathConstants.STRING); - return StringUtils.isBlank(value) ? null : value.trim(); - } catch (XPathExpressionException e) { - return null; + NodeList detailNodes = (NodeList) xpath.evaluate( + ".//*[local-name()='Details']/*[local-name()='Detail']", + metadataSection, + XPathConstants.NODESET + ); + + for (int i = 0; i < detailNodes.getLength(); i++) { + Node detailNode = detailNodes.item(i); + String key = xpathString(xpath, detailNode, "./*[local-name()='Key']/text()"); + if (StringUtils.isBlank(key)) { + continue; + } + String value = xpathString(xpath, detailNode, "./*[local-name()='Value']/text()"); + details.put(key, defaultString(value)); + } + } catch (XPathExpressionException ignored) { + } + if (!details.isEmpty()) { + vm.setDetails(details); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index ccf496db192c..b939224d8740 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.veeam.api.dto; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.ApiConstants; @@ -79,6 +80,9 @@ public final class Vm extends BaseDto { // CloudStack-specific fields private String accountId; + private String affinityGroupId; + private String userDataId; + private Map details; public String getName() { return name; @@ -297,6 +301,33 @@ public void setAccountId(String accountId) { this.accountId = accountId; } + @JsonIgnore + public String getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } + + @JsonIgnore + public String getUserDataId() { + return userDataId; + } + + public void setUserDataId(String userDataId) { + this.userDataId = userDataId; + } + + @JsonIgnore + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + @JsonInclude(JsonInclude.Include.NON_NULL) public static final class Bios { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java new file mode 100644 index 000000000000..8a21b595b770 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.api.request; + +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +public class ListQuery { + boolean allContent; + Long max; + Long page; + Map search; + + public boolean isAllContent() { + return allContent; + } + + public void setAllContent(boolean allContent) { + this.allContent = allContent; + } + + public Long getMax() { + return max; + } + + public void setMax(Long max) { + this.max = max; + } + + public Map getSearch() { + return search; + } + + public void setSearch(Map search) { + this.search = search; + } + + public Long getPage() { + return page; + } + + public Long getOffset() { + if (page == null || max == null) { + return null; + } + return Math.max(0, (page - 1)) * max; + } + + public Long getLimit() { + return max; + } + + public static ListQuery fromRequest(HttpServletRequest request) { + ListQuery query = new ListQuery(); + if (MapUtils.isEmpty(request.getParameterMap())) { + return query; + } + + String allContent = request.getParameter("all_content"); + if (StringUtils.isNotBlank(allContent)) { + query.setAllContent(Boolean.parseBoolean(allContent)); + } + String max = request.getParameter("max"); + if (StringUtils.isNotBlank(max)) { + try { + query.setMax(Long.parseLong(max)); + } catch (NumberFormatException e) { + // Ignore invalid max and keep default null value. + } + } + Map searchItems = getSearchMap(request.getParameter("search")); + if (!searchItems.isEmpty()) { + try { + query.setMax(Long.parseLong(searchItems.get("page"))); + } catch (NumberFormatException e) { + // Ignore invalid page and keep default null value. + } + query.setSearch(searchItems); + } + + return query; + } + + // Parse search clause. Only keep items which use simple '=' operator, and ignore others. For example: + // name=myvm and status=up --> {name=myvm, status=up} + // name=myvm and status!=down --> {name=myvm} (ignore status!=down because it uses '!=' operator) + @NotNull + private static Map getSearchMap(String searchClause) { + Map searchItems = new LinkedHashMap<>(); + if (StringUtils.isBlank(searchClause)) { + return searchItems; + } + String[] terms = searchClause.trim().split("(?i)\\s+and\\s+"); + for (String term : terms) { + if (term == null) { + continue; + } + String trimmedTerm = term.trim(); + if (trimmedTerm.isEmpty()) { + continue; + } + + int eqIdx = trimmedTerm.indexOf('='); + if (eqIdx <= 0 || eqIdx != trimmedTerm.lastIndexOf('=')) { + continue; + } + char prev = trimmedTerm.charAt(eqIdx - 1); + if (prev == '!' || prev == '<' || prev == '>') { + continue; + } + + String key = trimmedTerm.substring(0, eqIdx).trim(); + String value = trimmedTerm.substring(eqIdx + 1).trim(); + if (!key.isEmpty() && !value.isEmpty()) { + searchItems.put(key, value); + } + } + return searchItems; + } +} diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java index 005e324cd710..acce4b7426ae 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDao.java @@ -26,6 +26,7 @@ import com.cloud.api.query.vo.HostJoinVO; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface HostJoinDao extends GenericDao { @@ -42,6 +43,6 @@ public interface HostJoinDao extends GenericDao { List findByClusterId(Long clusterId, Host.Type type); - List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType); + List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index be3598f9cc20..6d3174d94325 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -55,6 +55,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StorageStats; import com.cloud.user.AccountManager; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -414,7 +415,7 @@ private String calculateResourceAllocatedPercentage(float resource, float resour } @Override - public List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType) { + public List listRoutingHostsByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); @@ -423,6 +424,6 @@ public List listRoutingHostsByHypervisor(Hypervisor.HypervisorType h SearchCriteria sc = sb.create(); sc.setParameters("type", Host.Type.Routing); sc.setParameters("hypervisorType", hypervisorType); - return listBy(sc); + return listBy(sc, filter); } } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java index bc19e0892057..dc19d8481933 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java @@ -23,6 +23,7 @@ import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.storage.StoragePool; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -44,4 +45,6 @@ public interface StoragePoolJoinDao extends GenericDao List findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags); + List listByZoneAndProvider(long zoneId, Filter filter); + } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 8bfce47b1204..35651f657941 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -49,6 +49,7 @@ import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.user.AccountManager; import com.cloud.utils.StringUtils; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -410,4 +411,13 @@ public List findStoragePoolByScopeAndRuleTags(Long datacenterId, return filteredPools; } + @Override + public List listByZoneAndProvider(long zoneId, Filter filter) { + SearchBuilder sb = createSearchBuilder(); + sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("zoneId", zoneId); + return listBy(sc, filter); + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java index 351e367e8d05..55d65df7ffb3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java @@ -20,6 +20,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.user.Account; import com.cloud.uservm.UserVm; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.api.ApiConstants.VMDetails; @@ -51,5 +52,5 @@ List listByAccountServiceOfferingTemplateAndNotInState(long accoun List listLeaseInstancesExpiringInDays(int days); - List listByHypervisorType(Hypervisor.HypervisorType hypervisorType); + List listByHypervisorType(Hypervisor.HypervisorType hypervisorType, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 39b2b9b94218..d243bb7a5468 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -84,6 +84,7 @@ import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -498,7 +499,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us } if (userVm.getUserDataId() != null) { - userVmResponse.setUserDataId(userVm.getUserDataUUid()); + userVmResponse.setUserDataId(userVm.getUserDataUuid()); userVmResponse.setUserDataName(userVm.getUserDataName()); userVmResponse.setUserDataDetails(userVm.getUserDataDetails()); userVmResponse.setUserDataPolicy(userVm.getUserDataPolicy()); @@ -835,12 +836,12 @@ public List listLeaseInstancesExpiringInDays(int days) { } @Override - public List listByHypervisorType(Hypervisor.HypervisorType hypervisorType) { + public List listByHypervisorType(Hypervisor.HypervisorType hypervisorType, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("hypervisorType", sb.entity().getHypervisorType(), Op.EQ); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("hypervisorType", hypervisorType); - return listBy(sc); + return listBy(sc, filter); } } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java index e61ad1d8e2d2..c3b5859120fb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java @@ -24,6 +24,7 @@ import com.cloud.api.query.vo.VolumeJoinVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Volume; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface VolumeJoinDao extends GenericDao { @@ -38,5 +39,5 @@ public interface VolumeJoinDao extends GenericDao { List listByInstanceId(long instanceId); - List listByHypervisor(Hypervisor.HypervisorType hypervisorType); + List listByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 0261398a2326..20b6d69c5917 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -43,6 +43,7 @@ import com.cloud.user.AccountManager; import com.cloud.user.VmDiskStatisticsVO; import com.cloud.user.dao.VmDiskStatisticsDao; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.VirtualMachine; @@ -381,7 +382,7 @@ public List listByInstanceId(long instanceId) { } @Override - public List listByHypervisor(Hypervisor.HypervisorType hypervisorType) { + public List listByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("vmType", sb.entity().getVmType(), SearchCriteria.Op.EQ); sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); @@ -389,7 +390,7 @@ public List listByHypervisor(Hypervisor.HypervisorType hypervisorT SearchCriteria sc = sb.create(); sc.setParameters("vmType", VirtualMachine.Type.User); sc.setParameters("hypervisorType", hypervisorType); - return search(sc, null); + return search(sc, filter); } } diff --git a/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java index 0b60d99adc2c..94549878b9fe 100644 --- a/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java @@ -429,7 +429,7 @@ public class UserVmJoinVO extends BaseViewWithTagInformationVO implements Contro private int jobStatus; @Column(name = "affinity_group_id") - private long affinityGroupId; + private Long affinityGroupId; @Column(name = "affinity_group_uuid") private String affinityGroupUuid; @@ -1012,7 +1012,7 @@ public String getIp6Cidr() { return ip6Cidr; } - public long getAffinityGroupId() { + public Long getAffinityGroupId() { return affinityGroupId; } @@ -1057,7 +1057,7 @@ public Long getUserDataId() { return userDataId; } - public String getUserDataUUid() { + public String getUserDataUuid() { return userDataUuid; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 4594ca6301fe..419e80ea9ada 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -67,6 +68,8 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.AccountService; +import com.cloud.user.User; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -104,6 +107,9 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject private PrimaryDataStoreDao primaryDataStoreDao; + @Inject + AccountService accountService; + private Timer imageTransferTimer; private boolean isKVMBackupExportServiceSupported(Long zoneId) { @@ -493,8 +499,10 @@ public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) { @Override public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format) { + User callingUser = CallContext.current().getCallingUser(); ImageTransfer imageTransfer; VolumeVO volume = volumeDao.findById(volumeId); + accountService.checkAccess(callingUser, volume); if (volume == null) { throw new CloudRuntimeException("Volume not found with the specified Id"); diff --git a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java index cf71d74498fc..4c646b5264b6 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java @@ -28,6 +28,7 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; @@ -160,13 +161,18 @@ public boolean update(final Long networkId, final NetworkVO network, final Map listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType, Filter filter) { + return null; + } + @Override public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { return null; } @Override - public List listByTrafficType(final TrafficType trafficType) { + public List listByTrafficType(final TrafficType trafficType, Filter filter) { return null; } From 414d96e70c420bf1c8ca9cf81b3e551d5baf0b34 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Apr 2026 11:18:50 +0530 Subject: [PATCH 086/129] remove unused classes Signed-off-by: Abhishek Kumar --- .../veeam/api/request/VmListQuery.java | 106 ------- .../veeam/api/request/VmSearchExpr.java | 102 ------- .../veeam/api/request/VmSearchFilters.java | 62 ---- .../veeam/api/request/VmSearchParser.java | 274 ------------------ 4 files changed, 544 deletions(-) delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java deleted file mode 100644 index 9383979c2b72..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmListQuery.java +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.request; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Query parameters supported by GET /api/vms (oVirt-like). - * - * Examples: - * /api/vms?search=name=myvm&max=50&page=1 - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public final class VmListQuery { - - /** - * oVirt-like search expression, e.g.: - * name=myvm - * status=down - * name=myvm and status=up - */ - @JsonProperty("search") - private String search; - - /** - * Max number of entries to return. - */ - @JsonProperty("max") - private Integer max; - - /** - * 1-based page number. - */ - @JsonProperty("page") - private Integer page; - - public VmListQuery() { - } - - public VmListQuery(final String search, final Integer max, final Integer page) { - this.search = search; - this.max = max; - this.page = page; - } - - public String getSearch() { - return search; - } - - public void setSearch(final String search) { - this.search = search; - } - - public Integer getMax() { - return max; - } - - public void setMax(final Integer max) { - this.max = max; - } - - public Integer getPage() { - return page; - } - - public void setPage(final Integer page) { - this.page = page; - } - - // ----- helpers (optional, but convenient) ----- - - @JsonIgnore - public int resolvedMax(final int defaultMax, final int hardCap) { - final int m = (max == null || max <= 0) ? defaultMax : max; - return Math.min(m, hardCap); - } - - @JsonIgnore - public int resolvedPage(final int defaultPage) { - return (page == null || page <= 0) ? defaultPage : page; - } - - @JsonIgnore - public int offset(final int defaultMax, final int hardCap, final int defaultPage) { - final int p = resolvedPage(defaultPage); - final int m = resolvedMax(defaultMax, hardCap); - return (p - 1) * m; - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java deleted file mode 100644 index 017fd9028598..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchExpr.java +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.request; - -import java.util.Objects; - -/** - * Small AST for oVirt-like search. - * - * Supported grammar: - * expr := orExpr - * orExpr := andExpr (OR andExpr)* - * andExpr := primary (AND primary)* - * primary := '(' expr ')' | term - * term := IDENT '=' (IDENT | STRING) - */ -public interface VmSearchExpr { - - final class Term implements VmSearchExpr { - private final String field; - private final String value; - - public Term(final String field, final String value) { - this.field = Objects.requireNonNull(field, "field"); - this.value = Objects.requireNonNull(value, "value"); - } - - public String getField() { - return field; - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return "Term(" + field + "=" + value + ")"; - } - } - - final class And implements VmSearchExpr { - private final VmSearchExpr left; - private final VmSearchExpr right; - - public And(final VmSearchExpr left, final VmSearchExpr right) { - this.left = Objects.requireNonNull(left, "left"); - this.right = Objects.requireNonNull(right, "right"); - } - - public VmSearchExpr getLeft() { - return left; - } - - public VmSearchExpr getRight() { - return right; - } - - @Override - public String toString() { - return "And(" + left + ", " + right + ")"; - } - } - - final class Or implements VmSearchExpr { - private final VmSearchExpr left; - private final VmSearchExpr right; - - public Or(final VmSearchExpr left, final VmSearchExpr right) { - this.left = Objects.requireNonNull(left, "left"); - this.right = Objects.requireNonNull(right, "right"); - } - - public VmSearchExpr getLeft() { - return left; - } - - public VmSearchExpr getRight() { - return right; - } - - @Override - public String toString() { - return "Or(" + left + ", " + right + ")"; - } - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java deleted file mode 100644 index 7cf12c0e32c9..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchFilters.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.request; - -import java.util.LinkedHashMap; -import java.util.Map; - -public final class VmSearchFilters { - - private final Map equals = new LinkedHashMap<>(); - - public Map equals() { - return equals; - } - - public VmSearchFilters put(final String field, final String value) { - equals.put(field, value); - return this; - } - - public static VmSearchFilters fromAndOnly(final VmSearchExpr expr) { - final VmSearchFilters f = new VmSearchFilters(); - if (expr == null) { - return f; - } - collect(expr, f); - return f; - } - - private static void collect(final VmSearchExpr expr, final VmSearchFilters f) { - if (expr instanceof VmSearchExpr.Term) { - final VmSearchExpr.Term t = (VmSearchExpr.Term) expr; - f.put(t.getField(), t.getValue()); - return; - } - if (expr instanceof VmSearchExpr.And) { - final VmSearchExpr.And a = (VmSearchExpr.And) expr; - collect(a.getLeft(), f); - collect(a.getRight(), f); - return; - } - if (expr instanceof VmSearchExpr.Or) { - throw new VmSearchParser.VmSearchParseException("Only AND expressions are supported currently"); - } - throw new VmSearchParser.VmSearchParseException("Unsupported search expression: " + expr.getClass().getName()); - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java deleted file mode 100644 index e8575750db48..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/VmSearchParser.java +++ /dev/null @@ -1,274 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.request; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -/** - * Parser for an oVirt-like 'search' parameter. - * - * Examples: - * name=myvm - * status=down and cluster=Default - * name="My VM" or name="Other VM" - * (status=up and host=hv1) or (status=down and host=hv2) - * - * Values can be IDENT (unquoted) or STRING (quoted with " ... "). - */ -public final class VmSearchParser { - - public static final class VmSearchParseException extends RuntimeException { - public VmSearchParseException(final String message) { super(message); } - } - - private final Set allowedFields; - - public VmSearchParser(final Set allowedFields) { - this.allowedFields = allowedFields; - } - - /** - * @return AST or null if input is null/blank - */ - public VmSearchExpr parse(final String input) { - if (input == null || input.trim().isEmpty()) { - return null; - } - final Lexer lexer = new Lexer(input); - final List tokens = lexer.lex(); - final Parser p = new Parser(tokens, allowedFields); - final VmSearchExpr expr = p.parseExpression(); - p.expect(TokenType.EOF); - return expr; - } - - // -------------------- lexer -------------------- - - enum TokenType { - IDENT, STRING, EQ, AND, OR, LPAREN, RPAREN, EOF - } - - static final class Token { - private final TokenType type; - private final String text; - private final int pos; - - Token(final TokenType type, final String text, final int pos) { - this.type = type; - this.text = text; - this.pos = pos; - } - - TokenType type() { return type; } - String text() { return text; } - int pos() { return pos; } - } - - static final class Lexer { - private final String s; - private final int n; - private int i = 0; - - Lexer(final String s) { - this.s = s; - this.n = s.length(); - } - - List lex() { - final List out = new ArrayList<>(); - while (true) { - skipWs(); - if (i >= n) { - out.add(new Token(TokenType.EOF, "", i)); - return out; - } - final char c = s.charAt(i); - - if (c == '(') { - out.add(new Token(TokenType.LPAREN, "(", i++)); - } else if (c == ')') { - out.add(new Token(TokenType.RPAREN, ")", i++)); - } else if (c == '=') { - out.add(new Token(TokenType.EQ, "=", i++)); - } else if (c == '"') { - out.add(readQuoted()); - } else if (isIdentStart(c)) { - out.add(readIdentOrKeyword()); - } else { - throw new VmSearchParseException("Unexpected character '" + c + "' at position " + i); - } - } - } - - private void skipWs() { - while (i < n) { - final char c = s.charAt(i); - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') i++; - else break; - } - } - - private Token readQuoted() { - final int start = i; - i++; // skip opening " - final StringBuilder b = new StringBuilder(); - while (i < n) { - final char c = s.charAt(i); - if (c == '"') { - i++; // closing " - return new Token(TokenType.STRING, b.toString(), start); - } - if (c == '\\') { - if (i + 1 >= n) { - throw new VmSearchParseException("Unterminated escape at position " + i); - } - final char nxt = s.charAt(i + 1); - switch (nxt) { - case '"': b.append('"'); i += 2; break; - case '\\': b.append('\\'); i += 2; break; - case 'n': b.append('\n'); i += 2; break; - case 't': b.append('\t'); i += 2; break; - default: - throw new VmSearchParseException("Unsupported escape \\" + nxt + " at position " + i); - } - continue; - } - b.append(c); - i++; - } - throw new VmSearchParseException("Unterminated string starting at position " + start); - } - - private Token readIdentOrKeyword() { - final int start = i; - i++; - while (i < n && isIdentPart(s.charAt(i))) i++; - - final String text = s.substring(start, i); - final String lower = text.toLowerCase(Locale.ROOT); - - if ("and".equals(lower)) return new Token(TokenType.AND, text, start); - if ("or".equals(lower)) return new Token(TokenType.OR, text, start); - - return new Token(TokenType.IDENT, text, start); - } - - private static boolean isIdentStart(final char c) { - return Character.isLetter(c) || c == '_' || c == '.'; - } - - private static boolean isIdentPart(final char c) { - return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == '-'; - } - } - - // -------------------- parser -------------------- - - static final class Parser { - private final List tokens; - private final Set allowedFields; - private int k = 0; - - Parser(final List tokens, final Set allowedFields) { - this.tokens = tokens; - this.allowedFields = allowedFields; - } - - VmSearchExpr parseExpression() { - return parseOr(); - } - - private VmSearchExpr parseOr() { - VmSearchExpr left = parseAnd(); - while (peek(TokenType.OR)) { - consume(TokenType.OR); - final VmSearchExpr right = parseAnd(); - left = new VmSearchExpr.Or(left, right); - } - return left; - } - - private VmSearchExpr parseAnd() { - VmSearchExpr left = parsePrimary(); - while (peek(TokenType.AND)) { - consume(TokenType.AND); - final VmSearchExpr right = parsePrimary(); - left = new VmSearchExpr.And(left, right); - } - return left; - } - - private VmSearchExpr parsePrimary() { - if (peek(TokenType.LPAREN)) { - consume(TokenType.LPAREN); - final VmSearchExpr e = parseExpression(); - expect(TokenType.RPAREN); - return e; - } - return parseTerm(); - } - - private VmSearchExpr parseTerm() { - final Token fieldTok = expect(TokenType.IDENT); - final String field = fieldTok.text(); - - if (allowedFields != null && !allowedFields.contains(field)) { - throw new VmSearchParseException("Unsupported search field '" + field + "' at position " + fieldTok.pos()); - } - - expect(TokenType.EQ); - - final Token v = next(); - final String value; - if (v.type() == TokenType.IDENT || v.type() == TokenType.STRING) { - value = v.text(); - } else { - throw new VmSearchParseException("Expected value after '=' at position " + v.pos()); - } - - if (value == null || value.isEmpty()) { - throw new VmSearchParseException("Empty value for field '" + field + "' at position " + v.pos()); - } - - return new VmSearchExpr.Term(field, value); - } - - boolean peek(final TokenType t) { - return tokens.get(k).type() == t; - } - - Token next() { - return tokens.get(k++); - } - - Token expect(final TokenType t) { - final Token tok = next(); - if (tok.type() != t) { - throw new VmSearchParseException("Expected " + t + " at position " + tok.pos() + " but found " + tok.type()); - } - return tok; - } - - Token consume(final TokenType t) { - return expect(t); - } - } -} From bf856ab3f4fb4a05fb03a6dab20a3f1f6ce8674e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Apr 2026 13:28:30 +0530 Subject: [PATCH 087/129] fix serviceoffering custom offering Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/adapter/ServerAdapter.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index ae5eb6e0717d..bc59d50a43a7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -670,7 +670,7 @@ public Vm createInstance(Vm request) { } } - protected ServiceOffering getServiceOfferingFromRequest(com.cloud.dc.DataCenter zone, Account account, + protected ServiceOfferingVO getServiceOfferingFromRequest(com.cloud.dc.DataCenter zone, Account account, String uuid, int cpu, int memory) { if (StringUtils.isBlank(uuid)) { return null; @@ -712,7 +712,7 @@ protected ServiceOffering getServiceOfferingFromRequest(com.cloud.dc.DataCenter protected ServiceOffering getServiceOfferingIdForVmCreation(com.cloud.dc.DataCenter zone, Account account, String serviceOfferingUuid, int cpu, int memory) { - ServiceOffering offering = getServiceOfferingFromRequest(zone, account, serviceOfferingUuid, cpu, memory); + ServiceOfferingVO offering = getServiceOfferingFromRequest(zone, account, serviceOfferingUuid, cpu, memory); if (offering != null) { return offering; } @@ -726,7 +726,12 @@ protected ServiceOffering getServiceOfferingIdForVmCreation(com.cloud.dc.DataCen return null; } String uuid = offerings.getResponses().get(0).getId(); - return serviceOfferingDao.findByUuid(uuid); + offering = serviceOfferingDao.findByUuid(uuid); + if (offering.isCustomized()) { + offering.setCpu(cpu); + offering.setRamSize(memory); + } + return offering; } protected VMTemplateVO getTemplateForInstanceCreation(String templateUuid) { @@ -803,7 +808,7 @@ protected Vm createInstance(com.cloud.dc.DataCenter zone, Long clusterId, Accoun Map instanceDetails = getDetailsForInstanceCreation(userdata, serviceOffering, details); if (MapUtils.isNotEmpty(instanceDetails)) { Map> map = new HashMap<>(); - map.put(0, details); + map.put(0, instanceDetails); cmd.setDetails(map); } cmd.setBlankInstance(true); From 5fd1b85afe9fbf84c51a5c9c7a314ee2eb4bc14d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Apr 2026 18:25:18 +0530 Subject: [PATCH 088/129] return internal CA certificate Signed-off-by: Abhishek Kumar --- .../services/PkiResourceRouteHandler.java | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java index 0e2037ba9db0..24c63e085dfe 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java @@ -27,15 +27,16 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Base64; import java.util.Enumeration; +import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; @@ -51,6 +52,10 @@ public class PkiResourceRouteHandler extends ManagerBase implements RouteHandler private static final String FORMAT_KEY = "format"; private static final String FORMAT_VALUE = "X509-PEM-CA"; private static final Charset OUTPUT_CHARSET = StandardCharsets.ISO_8859_1; + private static final boolean USE_CA_CERTS = true; + + @Inject + CAManager caManager; @Override public boolean canHandle(String method, String path) { @@ -84,21 +89,11 @@ protected void handleGet(HttpServletRequest req, HttpServletResponse resp, return; } - final String keystorePath = ServerPropertiesUtil.getKeystoreFile(); - final String keystorePassword = ServerPropertiesUtil.getKeystorePassword(); - - Path path = Path.of(keystorePath); - if (keystorePath.isBlank() || !Files.exists(path)) { - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "CloudStack HTTPS keystore not found"); + byte[] pemBytes = USE_CA_CERTS ? returnCACertificate() : returnMSCertificate(); + if (pemBytes == null || pemBytes.length == 0) { + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "No certificate data available"); return; } - - final X509Certificate caCert = - extractCaFromKeystore(path, keystorePassword); - - // DER encoding → browser downloads as .cer (oVirt behavior) - final byte[] pemBytes = - toPem(caCert).getBytes(OUTPUT_CHARSET); resp.setStatus(HttpServletResponse.SC_OK); resp.setHeader("Cache-Control", "no-store"); resp.setContentType("application/x-x509-ca-cert; charset=" + OUTPUT_CHARSET.name()); @@ -116,6 +111,33 @@ protected void handleGet(HttpServletRequest req, HttpServletResponse resp, } } + private byte[] returnCACertificate() throws IOException { + String tlsCaCert = caManager.getCaCertificate(null); + return tlsCaCert.getBytes(OUTPUT_CHARSET); + } + + // ToDo: To be removed + private static byte[] returnMSCertificate() throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { + final String keystorePath = ServerPropertiesUtil.getKeystoreFile(); + final String keystorePassword = ServerPropertiesUtil.getKeystorePassword(); + + Path path = Path.of(keystorePath); + if (keystorePath.isBlank() || !Files.exists(path)) { + return null; + } + + final X509Certificate caCert = + extractCaFromKeystore(path, keystorePassword); + + // DER encoding → browser downloads as .cer (oVirt behavior) + String base64 = Base64.getMimeEncoder(64, new byte[]{'\n'}) + .encodeToString(caCert.getEncoded()); + String cert = "-----BEGIN CERTIFICATE-----\n" + + base64 + + "\n-----END CERTIFICATE-----\n"; + return cert.getBytes(OUTPUT_CHARSET); + } + private static X509Certificate extractCaFromKeystore(Path ksPath, String ksPassword) throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { @@ -162,12 +184,4 @@ private static X509Certificate extractCaFromKeystore(Path ksPath, String ksPassw return (X509Certificate) cert; } - - private static String toPem(X509Certificate cert) throws CertificateEncodingException { - String base64 = Base64.getMimeEncoder(64, new byte[]{'\n'}) - .encodeToString(cert.getEncoded()); - return "-----BEGIN CERTIFICATE-----\n" - + base64 - + "\n-----END CERTIFICATE-----\n"; - } } From 2d2f74078ffaf41a2fb2f8c45e5e20f3f3695870 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:48:52 +0530 Subject: [PATCH 089/129] Support local storage and shared mount point --- .../LibvirtStartBackupCommandWrapper.java | 1 - .../backup/KVMBackupExportServiceImpl.java | 61 ++++++++++++++----- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 4ed39f1ae895..2e7c8c5ae98c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -171,7 +171,6 @@ private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, } String diskName = entry.getValue(); String export = diskPathUuidMap.get(entry.getKey()); - // todo: use UUID here as well? String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; xml.append(" hosts = null; - if (storagePoolVO.getScope().equals(ScopeType.CLUSTER)) { - hosts = hostDao.findByClusterId(storagePoolVO.getClusterId()); - - } else if (storagePoolVO.getScope().equals(ScopeType.ZONE)) { - hosts = hostDao.findByDataCenterId(storagePoolVO.getDataCenterId()); + private HostVO getRandomHostFromStoragePool(StoragePoolVO storagePool) { + List hosts; + switch (storagePool.getScope()) { + case CLUSTER: + hosts = hostDao.findByClusterId(storagePool.getClusterId()); + Collections.shuffle(hosts); + return hosts.get(0); + case ZONE: + hosts = hostDao.findByDataCenterId(storagePool.getDataCenterId()); + Collections.shuffle(hosts); + return hosts.get(0); + case HOST: + List storagePoolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId()); + Collections.shuffle(storagePoolHostVOs); + return hostDao.findById(storagePoolHostVOs.get(0).getHostId()); + default: + throw new CloudRuntimeException("Unsupported storage pool scope: " + storagePool.getScope()); } - return hosts.get(0); } private void startNBDServer(String transferId, String direction, Long hostId, String exportName, String volumePath, String checkpointId) { @@ -396,12 +411,24 @@ private void startNBDServer(String transferId, String direction, Long hostId, St } } + private String getVolumePathPrefix(StoragePoolVO storagePool) { + if (ScopeType.HOST.equals(storagePool.getScope())) { + return storagePool.getPath(); + } + switch (storagePool.getPoolType()) { + case NetworkFilesystem: + return String.format("/mnt/%s", storagePool.getUuid()); + case SharedMountPoint: + return storagePool.getPath(); + default: + throw new CloudRuntimeException("Unsupported storage pool type for file based image transfer: " + storagePool.getPoolType()); + } + } + private String getVolumePathForFileBasedBackend(Volume volume) { - Long poolId = volume.getPoolId(); - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); - // todo: This only works with file based storage (not ceph, linbit) - String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); - return volumePath; + StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); + String volumePathPrefix = getVolumePathPrefix(storagePool); + return volumePathPrefix + "/" + volume.getPath(); } private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer.Backend backend) { @@ -409,8 +436,12 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer String transferId = UUID.randomUUID().toString(); Long poolId = volume.getPoolId(); - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); - Host host = getFirstHostFromStoragePool(storagePoolVO); + StoragePoolVO storagePool = poolId == null ? null : primaryDataStoreDao.findById(poolId); + if (storagePool == null) { + throw new CloudRuntimeException("Storage pool cannot be determined for volume: " + volume.getUuid()); + } + + Host host = getRandomHostFromStoragePool(storagePool); String volumePath = getVolumePathForFileBasedBackend(volume); ImageTransferVO imageTransfer; From cdf4684bcf9961ea5dfd1bc7f1029d7b32cddd70 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 2 Apr 2026 06:30:08 +0530 Subject: [PATCH 090/129] use shared=0 for unittests --- scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py index c322a9920477..2ae95d01f4b5 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py @@ -333,7 +333,7 @@ def start(self) -> None: "--socket", self.socket_path, "--format", self.image_format, "--persistent", - "--shared=8", + "--shared=0", "--cache=none", self.image_path, ], From 6f4758d062767f22ee4eb68c7de024cb469a9fb3 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 1 Apr 2026 12:08:30 +0530 Subject: [PATCH 091/129] expiry timeouts for idle image transfers --- .../backup/KVMBackupExportService.java | 6 + .../backup/CreateImageTransferCommand.java | 16 +- ...virtCreateImageTransferCommandWrapper.java | 1 + .../vm/hypervisor/kvm/imageserver/config.py | 118 +++++- .../hypervisor/kvm/imageserver/constants.py | 4 + .../vm/hypervisor/kvm/imageserver/handler.py | 348 +++++++++--------- .../vm/hypervisor/kvm/imageserver/server.py | 56 +-- .../kvm/imageserver/tests/test_base.py | 9 +- .../imageserver/tests/test_registry_idle.py | 100 +++++ .../tests/test_transfer_idle_expiry.py | 57 +++ .../backup/KVMBackupExportServiceImpl.java | 18 +- 11 files changed, 513 insertions(+), 220 deletions(-) create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index 7a53c1370c6e..fbbde961ad17 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -43,6 +43,12 @@ public interface KVMBackupExportService extends Configurable, PluggableService { "10", "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); + ConfigKey ImageTransferIdleTimeoutSeconds = new ConfigKey<>("Advanced", Integer.class, + "image.transfer.idle.timeout.seconds", + "600", + "Seconds since last completed HTTP request to an image transfer before the image server unregisters it (idle timeout).", + true, ConfigKey.Scope.Zone); + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, "expose.kvm.backup.export.service.apis", "false", diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 3e042bf42491..95b56c9a9c38 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -27,25 +27,27 @@ public class CreateImageTransferCommand extends Command { private String checkpointId; private String file; private ImageTransfer.Backend backend; + private int idleTimeoutSeconds; public CreateImageTransferCommand() { } - private CreateImageTransferCommand(String transferId, String direction, String socket) { + private CreateImageTransferCommand(String transferId, String direction, String socket, int idleTimeoutSeconds) { this.transferId = transferId; this.direction = direction; this.socket = socket; + this.idleTimeoutSeconds = idleTimeoutSeconds; } - public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId) { - this(transferId, direction, socket); + public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); this.backend = ImageTransfer.Backend.nbd; this.exportName = exportName; this.checkpointId = checkpointId; } - public CreateImageTransferCommand(String transferId, String direction, String socket, String file) { - this(transferId, direction, socket); + public CreateImageTransferCommand(String transferId, String direction, String socket, String file, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); if (direction == ImageTransfer.Direction.download.toString()) { throw new IllegalArgumentException("File backend is only supported for upload"); } @@ -85,4 +87,8 @@ public String getDirection() { public String getCheckpointId() { return checkpointId; } + + public int getIdleTimeoutSeconds() { + return idleTimeoutSeconds; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 1b9b33f83a9b..01fd11524bc7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -131,6 +131,7 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r final Map payload = new HashMap<>(); payload.put("backend", backend.toString()); + payload.put("idle_timeout_seconds", cmd.getIdleTimeoutSeconds()); if (backend == ImageTransfer.Backend.file) { final String filePath = cmd.getFile(); diff --git a/scripts/vm/hypervisor/kvm/imageserver/config.py b/scripts/vm/hypervisor/kvm/imageserver/config.py index 3b1fd686f053..98515d7519bb 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/config.py +++ b/scripts/vm/hypervisor/kvm/imageserver/config.py @@ -18,7 +18,60 @@ import logging import os import threading -from typing import Any, Dict, Optional +import time +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Optional + +from .constants import DEFAULT_IDLE_TIMEOUT_SECONDS + + +def parse_idle_timeout_seconds(obj: dict) -> int: + """Seconds of idle time (no completed HTTP requests) before unregister.""" + v = obj.get("idle_timeout_seconds", DEFAULT_IDLE_TIMEOUT_SECONDS) + if not isinstance(v, int): + raise ValueError("idle_timeout_seconds must be an integer") + v = int(v) + if v < 1: + v = 86400 * 7 + return v + + +def validate_transfer_config(obj: dict) -> dict: + """ + Validate and normalize a transfer config dict received over the control + socket. Returns the cleaned config or raises ValueError. + """ + idle_sec = parse_idle_timeout_seconds(obj) + + backend = obj.get("backend") + if backend is None: + backend = "nbd" + if not isinstance(backend, str): + raise ValueError("invalid backend type") + backend = backend.lower() + if backend not in ("nbd", "file"): + raise ValueError(f"unsupported backend: {backend}") + + if backend == "file": + file_path = obj.get("file") + if not isinstance(file_path, str) or not file_path.strip(): + raise ValueError("missing/invalid file path for file backend") + return {"backend": "file", "file": file_path.strip(), "idle_timeout_seconds": idle_sec} + + socket_path = obj.get("socket") + export = obj.get("export") + export_bitmap = obj.get("export_bitmap") + if not isinstance(socket_path, str) or not socket_path.strip(): + raise ValueError("missing/invalid socket path for nbd backend") + if export is not None and (not isinstance(export, str) or not export): + raise ValueError("invalid export name") + return { + "backend": "nbd", + "socket": socket_path.strip(), + "export": export, + "export_bitmap": export_bitmap, + "idle_timeout_seconds": idle_sec, + } def safe_transfer_id(image_id: str) -> Optional[str]: @@ -43,11 +96,17 @@ class TransferRegistry: The cloudstack-agent registers/unregisters transfers via the Unix domain control socket. The HTTP handler looks up configs through get(). + + Each transfer may specify idle_timeout_seconds (default DEFAULT_IDLE_TIMEOUT_SECONDS). + After no in-flight HTTP requests have completed for that idle period, the transfer + is removed (same effect as unregister). """ def __init__(self) -> None: self._lock = threading.Lock() self._transfers: Dict[str, Dict[str, Any]] = {} + self._last_activity: Dict[str, float] = {} + self._inflight: Dict[str, int] = {} def register(self, transfer_id: str, config: Dict[str, Any]) -> bool: safe_id = safe_transfer_id(transfer_id) @@ -56,6 +115,8 @@ def register(self, transfer_id: str, config: Dict[str, Any]) -> bool: return False with self._lock: self._transfers[safe_id] = config + self._last_activity[safe_id] = time.monotonic() + self._inflight.pop(safe_id, None) logging.info("registered transfer_id=%s active=%d", safe_id, len(self._transfers)) return True @@ -68,6 +129,8 @@ def unregister(self, transfer_id: str) -> int: return len(self._transfers) with self._lock: self._transfers.pop(safe_id, None) + self._last_activity.pop(safe_id, None) + self._inflight.pop(safe_id, None) remaining = len(self._transfers) logging.info("unregistered transfer_id=%s active=%d", safe_id, remaining) return remaining @@ -82,3 +145,56 @@ def get(self, transfer_id: str) -> Optional[Dict[str, Any]]: def active_count(self) -> int: with self._lock: return len(self._transfers) + + @contextmanager + def request_lifecycle(self, transfer_id: str) -> Iterator[None]: + """ + Track an HTTP request for idle-timeout purposes. + + Expiry is based on time since the last request *completed* (all in-flight + work for this transfer_id finished). Transfers with active requests are + never expired. + """ + safe_id = safe_transfer_id(transfer_id) + if safe_id is None: + yield + return + with self._lock: + if safe_id not in self._transfers: + yield + return + self._inflight[safe_id] = self._inflight.get(safe_id, 0) + 1 + try: + yield + finally: + now = time.monotonic() + with self._lock: + count = self._inflight.get(safe_id, 1) - 1 + if count <= 0: + self._inflight.pop(safe_id, None) + if safe_id in self._transfers: + self._last_activity[safe_id] = now + else: + self._inflight[safe_id] = count + + def sweep_expired_transfers(self) -> None: + """Remove transfers that exceeded idle_timeout_seconds with no in-flight HTTP work.""" + now = time.monotonic() + with self._lock: + expired: List[str] = [] + for tid, cfg in list(self._transfers.items()): + if self._inflight.get(tid, 0) > 0: + continue + timeout = int(cfg.get("idle_timeout_seconds", DEFAULT_IDLE_TIMEOUT_SECONDS)) + last = self._last_activity.get(tid, now) + if now - last >= timeout: + expired.append(tid) + for tid in expired: + self._transfers.pop(tid, None) + self._last_activity.pop(tid, None) + self._inflight.pop(tid, None) + logging.info( + "idle expiry: unregistered transfer_id=%s active=%d", + tid, + len(self._transfers), + ) diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py index 33cf3001d7a1..0b6465527f4b 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/constants.py +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -36,6 +36,10 @@ CONTROL_SOCKET_PERMISSIONS = 0o660 CONTROL_RECV_BUFFER = 4096 +# Transfer idle timeout (seconds). A transfer is expired when no in-flight HTTP +# requests have completed for this duration. +DEFAULT_IDLE_TIMEOUT_SECONDS = 600 + # Maximum size of a JSON body in a PATCH request (zero / flush ops) MAX_PATCH_JSON_SIZE = 64 * 1024 # 64 KiB diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index 9bfed8d52f93..c28a06575814 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -213,57 +213,58 @@ def do_OPTIONS(self) -> None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - backend = create_backend(cfg) - try: - if not backend.supports_extents: - allowed_methods = "GET, PUT, POST, OPTIONS" - features = ["flush"] + with self._registry.request_lifecycle(image_id): + backend = create_backend(cfg) + try: + if not backend.supports_extents: + allowed_methods = "GET, PUT, POST, OPTIONS" + features = ["flush"] + response = { + "unix_socket": None, + "features": features, + "max_readers": MAX_PARALLEL_READS, + "max_writers": MAX_PARALLEL_WRITES, + } + self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) + return + + read_only = True + can_flush = False + can_zero = False + try: + caps = backend.get_capabilities() + read_only = caps["read_only"] + can_flush = caps["can_flush"] + can_zero = caps["can_zero"] + except Exception as e: + logging.warning("OPTIONS: could not query backend capabilities: %r", e) + read_only = bool(cfg.get("read_only")) + if not read_only: + can_flush = True + can_zero = True + + if read_only: + allowed_methods = "GET, OPTIONS" + features = ["extents"] + max_writers = 0 + else: + allowed_methods = "GET, PUT, PATCH, OPTIONS" + features = ["extents"] + if can_zero: + features.append("zero") + if can_flush: + features.append("flush") + max_writers = MAX_PARALLEL_WRITES + response = { "unix_socket": None, "features": features, "max_readers": MAX_PARALLEL_READS, - "max_writers": MAX_PARALLEL_WRITES, + "max_writers": max_writers, } self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - return - - read_only = True - can_flush = False - can_zero = False - try: - caps = backend.get_capabilities() - read_only = caps["read_only"] - can_flush = caps["can_flush"] - can_zero = caps["can_zero"] - except Exception as e: - logging.warning("OPTIONS: could not query backend capabilities: %r", e) - read_only = bool(cfg.get("read_only")) - if not read_only: - can_flush = True - can_zero = True - - if read_only: - allowed_methods = "GET, OPTIONS" - features = ["extents"] - max_writers = 0 - else: - allowed_methods = "GET, PUT, PATCH, OPTIONS" - features = ["extents"] - if can_zero: - features.append("zero") - if can_flush: - features.append("flush") - max_writers = MAX_PARALLEL_WRITES - - response = { - "unix_socket": None, - "features": features, - "max_readers": MAX_PARALLEL_READS, - "max_writers": max_writers, - } - self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) - finally: - backend.close() + finally: + backend.close() def do_GET(self) -> None: image_id, tail = self._parse_route() @@ -277,25 +278,27 @@ def do_GET(self) -> None: return if tail == "extents": - backend = create_backend(cfg) - try: - if not backend.supports_extents: - self._send_error_json( - HTTPStatus.BAD_REQUEST, "extents not supported for file backend" - ) - return - finally: - backend.close() - query = self._parse_query() - context = (query.get("context") or [None])[0] - self._handle_get_extents(image_id, cfg, context=context) + with self._registry.request_lifecycle(image_id): + backend = create_backend(cfg) + try: + if not backend.supports_extents: + self._send_error_json( + HTTPStatus.BAD_REQUEST, "extents not supported for file backend" + ) + return + finally: + backend.close() + query = self._parse_query() + context = (query.get("context") or [None])[0] + self._handle_get_extents(image_id, cfg, context=context) return if tail is not None: self._send_error_json(HTTPStatus.NOT_FOUND, "not found") return range_header = self.headers.get("Range") - self._handle_get_image(image_id, cfg, range_header) + with self._registry.request_lifecycle(image_id): + self._handle_get_image(image_id, cfg, range_header) def do_PUT(self) -> None: image_id, tail = self._parse_route() @@ -308,46 +311,47 @@ def do_PUT(self) -> None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - if self.headers.get("Range") is not None: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "Range header not supported for PUT; use Content-Range or PATCH", - ) - return + with self._registry.request_lifecycle(image_id): + if self.headers.get("Range") is not None: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "Range header not supported for PUT; use Content-Range or PATCH", + ) + return - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return - query = self._parse_query() - flush_param = (query.get("flush") or ["n"])[0].lower() - flush = flush_param in ("y", "yes", "true", "1") + query = self._parse_query() + flush_param = (query.get("flush") or ["n"])[0].lower() + flush = flush_param in ("y", "yes", "true", "1") - content_range_hdr = self.headers.get("Content-Range") - if content_range_hdr is not None: - backend = create_backend(cfg) - try: - if not backend.supports_range_write: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "Content-Range PUT not supported for file backend; use full PUT", - ) - return - finally: - backend.close() - self._handle_put_range(image_id, cfg, content_range_hdr, content_length, flush) - return + content_range_hdr = self.headers.get("Content-Range") + if content_range_hdr is not None: + backend = create_backend(cfg) + try: + if not backend.supports_range_write: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "Content-Range PUT not supported for file backend; use full PUT", + ) + return + finally: + backend.close() + self._handle_put_range(image_id, cfg, content_range_hdr, content_length, flush) + return - self._handle_put_image(image_id, cfg, content_length, flush) + self._handle_put_image(image_id, cfg, content_length, flush) def do_POST(self) -> None: image_id, tail = self._parse_route() @@ -361,7 +365,8 @@ def do_POST(self) -> None: return if tail == "flush": - self._handle_post_flush(image_id, cfg) + with self._registry.request_lifecycle(image_id): + self._handle_post_flush(image_id, cfg) return self._send_error_json(HTTPStatus.NOT_FOUND, "not found") @@ -376,21 +381,44 @@ def do_PATCH(self) -> None: self._send_error_json(HTTPStatus.NOT_FOUND, "unknown image_id") return - backend = create_backend(cfg) - try: - if not backend.supports_range_write: + with self._registry.request_lifecycle(image_id): + backend = create_backend(cfg) + try: + if not backend.supports_range_write: + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "range writes and PATCH not supported for file backend; use PUT for full upload", + ) + return + finally: + backend.close() + + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + range_header = self.headers.get("Range") + + if range_header is not None and content_type != "application/json": + content_length_hdr = self.headers.get("Content-Length") + if content_length_hdr is None: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") + return + try: + content_length = int(content_length_hdr) + except ValueError: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return + if content_length <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") + return + self._handle_patch_range(image_id, cfg, range_header, content_length) + return + + if content_type != "application/json": self._send_error_json( - HTTPStatus.BAD_REQUEST, - "range writes and PATCH not supported for file backend; use PUT for full upload", + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", ) return - finally: - backend.close() - content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() - range_header = self.headers.get("Range") - - if range_header is not None and content_type != "application/json": content_length_hdr = self.headers.get("Content-Length") if content_length_hdr is None: self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") @@ -400,82 +428,60 @@ def do_PATCH(self) -> None: except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") return - if content_length <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length must be positive") + if content_length <= 0 or content_length > MAX_PATCH_JSON_SIZE: + self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") return - self._handle_patch_range(image_id, cfg, range_header, content_length) - return - - if content_type != "application/json": - self._send_error_json( - HTTPStatus.UNSUPPORTED_MEDIA_TYPE, - "PATCH requires Content-Type: application/json (for zero/flush) or Range with binary body", - ) - return - content_length_hdr = self.headers.get("Content-Length") - if content_length_hdr is None: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Content-Length required") - return - try: - content_length = int(content_length_hdr) - except ValueError: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - if content_length <= 0 or content_length > MAX_PATCH_JSON_SIZE: - self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") - return - - body = self.rfile.read(content_length) - if len(body) != content_length: - self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") - return - - try: - payload = json.loads(body.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError) as e: - self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") - return + body = self.rfile.read(content_length) + if len(body) != content_length: + self._send_error_json(HTTPStatus.BAD_REQUEST, "request body truncated") + return - if not isinstance(payload, dict): - self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") - return + try: + payload = json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self._send_error_json(HTTPStatus.BAD_REQUEST, f"invalid JSON: {e}") + return - op = payload.get("op") - if op == "flush": - self._handle_post_flush(image_id, cfg) - return - if op != "zero": - self._send_error_json( - HTTPStatus.BAD_REQUEST, - "unsupported op; only \"zero\" and \"flush\" are supported", - ) - return + if not isinstance(payload, dict): + self._send_error_json(HTTPStatus.BAD_REQUEST, "body must be a JSON object") + return - try: - size = int(payload.get("size")) - except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") - return - if size <= 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") - return + op = payload.get("op") + if op == "flush": + self._handle_post_flush(image_id, cfg) + return + if op != "zero": + self._send_error_json( + HTTPStatus.BAD_REQUEST, + "unsupported op; only \"zero\" and \"flush\" are supported", + ) + return - offset = payload.get("offset") - if offset is None: - offset = 0 - else: try: - offset = int(offset) + size = int(payload.get("size")) except (TypeError, ValueError): - self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") + self._send_error_json(HTTPStatus.BAD_REQUEST, "missing or invalid \"size\"") return - if offset < 0: - self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") + if size <= 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"size\" must be positive") return - flush = bool(payload.get("flush", False)) - self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) + offset = payload.get("offset") + if offset is None: + offset = 0 + else: + try: + offset = int(offset) + except (TypeError, ValueError): + self._send_error_json(HTTPStatus.BAD_REQUEST, "invalid \"offset\"") + return + if offset < 0: + self._send_error_json(HTTPStatus.BAD_REQUEST, "\"offset\" must be non-negative") + return + + flush = bool(payload.get("flush", False)) + self._handle_patch_zero(image_id, cfg, offset=offset, size=size, flush=flush) # ------------------------------------------------------------------ # Operation handlers diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index 99318bc58fc4..1bc42252d4f2 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -22,6 +22,7 @@ import socket import ssl import threading +import time from http.server import HTTPServer from socketserver import ThreadingMixIn from typing import Type @@ -33,7 +34,7 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] pass from .concurrency import ConcurrencyManager -from .config import TransferRegistry +from .config import TransferRegistry, validate_transfer_config from .constants import ( CONTROL_RECV_BUFFER, CONTROL_SOCKET, @@ -65,41 +66,6 @@ class ConfiguredHandler(Handler): return ConfiguredHandler -def _validate_config(obj: dict) -> dict: - """ - Validate and normalize a transfer config dict received over the control - socket. Returns the cleaned config or raises ValueError. - """ - backend = obj.get("backend") - if backend is None: - backend = "nbd" - if not isinstance(backend, str): - raise ValueError("invalid backend type") - backend = backend.lower() - if backend not in ("nbd", "file"): - raise ValueError(f"unsupported backend: {backend}") - - if backend == "file": - file_path = obj.get("file") - if not isinstance(file_path, str) or not file_path.strip(): - raise ValueError("missing/invalid file path for file backend") - return {"backend": "file", "file": file_path.strip()} - - socket_path = obj.get("socket") - export = obj.get("export") - export_bitmap = obj.get("export_bitmap") - if not isinstance(socket_path, str) or not socket_path.strip(): - raise ValueError("missing/invalid socket path for nbd backend") - if export is not None and (not isinstance(export, str) or not export): - raise ValueError("invalid export name") - return { - "backend": "nbd", - "socket": socket_path.strip(), - "export": export, - "export_bitmap": export_bitmap, - } - - def _handle_control_conn(conn: socket.socket, registry: TransferRegistry) -> None: """Handle a single control-socket connection (one JSON request/response).""" try: @@ -122,7 +88,7 @@ def _handle_control_conn(conn: socket.socket, registry: TransferRegistry) -> Non resp = {"status": "error", "message": "missing transfer_id or config"} else: try: - config = _validate_config(raw_config) + config = validate_transfer_config(raw_config) except ValueError as e: resp = {"status": "error", "message": str(e)} else: @@ -153,6 +119,15 @@ def _handle_control_conn(conn: socket.socket, registry: TransferRegistry) -> Non conn.close() +def _idle_sweep_loop(registry: TransferRegistry, interval_s: float = 10.0) -> None: + while True: + time.sleep(interval_s) + try: + registry.sweep_expired_transfers() + except Exception: + logging.exception("idle sweep error") + + def _control_listener(registry: TransferRegistry, sock_path: str) -> None: """Accept loop for the Unix domain control socket (runs in a daemon thread).""" if os.path.exists(sock_path): @@ -221,6 +196,13 @@ def main() -> None: ) ctrl_thread.start() + sweep_thread = threading.Thread( + target=_idle_sweep_loop, + args=(registry,), + daemon=True, + ) + sweep_thread.start() + addr = (args.listen, args.port) httpd = ThreadingHTTPServer(addr, handler_cls) diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py index 2ae95d01f4b5..c8703f8a1082 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py @@ -374,18 +374,23 @@ def make_tmp_image(data=None, image_size=IMAGE_SIZE) -> str: return path -def make_file_transfer(data=None, image_size=IMAGE_SIZE): +def make_file_transfer(data=None, image_size=IMAGE_SIZE, idle_timeout_seconds=None): """ Create a temp file + register a file-backend transfer. Returns (transfer_id, url, file_path, cleanup_callable). + + If *idle_timeout_seconds* is set, it is sent in the transfer config (for idle expiry tests). """ srv = get_image_server() path = make_tmp_image(data=data, image_size=image_size) transfer_id = f"file-{uuid.uuid4().hex[:8]}" + cfg = {"backend": "file", "file": path} + if idle_timeout_seconds is not None: + cfg["idle_timeout_seconds"] = idle_timeout_seconds resp = srv["send"]({ "action": "register", "transfer_id": transfer_id, - "config": {"backend": "file", "file": path}, + "config": cfg, }) assert resp["status"] == "ok", f"register failed: {resp}" url = f"{srv['base_url']}/images/{transfer_id}" diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py new file mode 100644 index 000000000000..7fa959416613 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for transfer idle timeout (no image server / nbd dependency).""" + +import unittest +from unittest.mock import patch + +from imageserver.config import ( + TransferRegistry, + parse_idle_timeout_seconds, + validate_transfer_config, +) +from imageserver.constants import DEFAULT_IDLE_TIMEOUT_SECONDS + + +class TestParseIdleTimeout(unittest.TestCase): + def test_default_600(self): + self.assertEqual(parse_idle_timeout_seconds({}), DEFAULT_IDLE_TIMEOUT_SECONDS) + + def test_explicit(self): + self.assertEqual( + parse_idle_timeout_seconds({"idle_timeout_seconds": 30}), 30 + ) + + def test_rejects_zero(self): + with self.assertRaises(ValueError): + parse_idle_timeout_seconds({"idle_timeout_seconds": 0}) + + +class TestValidateTransferConfig(unittest.TestCase): + def test_file_merges_idle(self): + c = validate_transfer_config( + {"backend": "file", "file": "/tmp/x", "idle_timeout_seconds": 3} + ) + self.assertEqual(c["idle_timeout_seconds"], 3) + self.assertEqual(c["backend"], "file") + + +class TestRegistryIdleSweep(unittest.TestCase): + def test_sweep_unregisters_after_idle(self): + clock = [0.0] + + def mono(): + return clock[0] + + with patch("imageserver.config.time.monotonic", mono): + r = TransferRegistry() + r.register( + "t1", + validate_transfer_config( + {"backend": "file", "file": "/x", "idle_timeout_seconds": 2} + ), + ) + clock[0] = 5.0 + r.sweep_expired_transfers() + self.assertIsNone(r.get("t1")) + + def test_inflight_prevents_sweep_until_request_ends(self): + clock = [0.0] + + def mono(): + return clock[0] + + with patch("imageserver.config.time.monotonic", mono): + r = TransferRegistry() + r.register( + "t1", + validate_transfer_config( + {"backend": "file", "file": "/x", "idle_timeout_seconds": 2} + ), + ) + clock[0] = 1.0 + ctx = r.request_lifecycle("t1") + ctx.__enter__() + clock[0] = 100.0 + r.sweep_expired_transfers() + self.assertIsNotNone(r.get("t1")) + ctx.__exit__(None, None, None) + clock[0] = 103.0 + r.sweep_expired_transfers() + self.assertIsNone(r.get("t1")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py new file mode 100644 index 000000000000..0cfbfc40ee9e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Integration tests for per-transfer HTTP idle timeout (requires image server deps e.g. nbd).""" + +import time +import urllib.error + +from .test_base import ( + ImageServerTestCase, + http_options, + make_file_transfer, +) + + +class TestTransferIdleExpiry(ImageServerTestCase): + def test_transfer_expires_after_idle(self): + """No HTTP activity after registration: transfer is unregistered after idle_timeout_seconds.""" + _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=2) + try: + time.sleep(3.5) + with self.assertRaises(urllib.error.HTTPError) as ctx: + http_options(url) + self.assertEqual(ctx.exception.code, 404) + st = self.ctrl({"action": "status"}) + self.assertEqual(st.get("status"), "ok") + finally: + cleanup() + + def test_http_activity_resets_idle_deadline(self): + """Completing a request resets the idle timer; transfer stays past a single interval.""" + _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=2) + try: + http_options(url) + time.sleep(1.2) + http_options(url) + time.sleep(1.2) + http_options(url) + time.sleep(1.2) + resp = http_options(url) + self.assertEqual(resp.status, 200) + finally: + cleanup() diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 9ddf8099c48e..d71f7b668480 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -327,12 +327,18 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu socket = transferId; } + HostVO backupHost = hostDao.findById(backup.getHostId()); + if (backupHost == null) { + throw new CloudRuntimeException("Host not found for backup: " + backupId); + } + int idleTimeoutSec = ImageTransferIdleTimeoutSeconds.valueIn(backupHost.getDataCenterId()); CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( transferId, direction, volume.getUuid(), socket, - backup.getFromCheckpointId()); + backup.getFromCheckpointId(), + idleTimeoutSec); try { CreateImageTransferAnswer answer; @@ -443,6 +449,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer Host host = getRandomHostFromStoragePool(storagePool); String volumePath = getVolumePathForFileBasedBackend(volume); + int idleTimeoutSec = ImageTransferIdleTimeoutSeconds.valueIn(host.getDataCenterId()); ImageTransferVO imageTransfer; CreateImageTransferCommand transferCmd; @@ -462,7 +469,8 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer transferId, direction, transferId, - volumePath); + volumePath, + idleTimeoutSec); } else { startNBDServer(transferId, direction, host.getId(), volume.getUuid(), volumePath, null); @@ -483,7 +491,8 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer direction, volume.getUuid(), transferId, - null); + null, + idleTimeoutSec); } CreateImageTransferAnswer transferAnswer; try { @@ -899,7 +908,8 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - ImageTransferPollingInterval + ImageTransferPollingInterval, + ImageTransferIdleTimeoutSeconds }; } } From 5310f2996aac287e8fdc0c99a5d860e591f94b19 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 1 Apr 2026 16:21:14 +0530 Subject: [PATCH 092/129] fix tests --- scripts/vm/hypervisor/kvm/imageserver/config.py | 2 +- .../kvm/imageserver/tests/test_registry_idle.py | 7 ++++--- .../imageserver/tests/test_transfer_idle_expiry.py | 12 ++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/config.py b/scripts/vm/hypervisor/kvm/imageserver/config.py index 98515d7519bb..1c92fd129379 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/config.py +++ b/scripts/vm/hypervisor/kvm/imageserver/config.py @@ -32,7 +32,7 @@ def parse_idle_timeout_seconds(obj: dict) -> int: raise ValueError("idle_timeout_seconds must be an integer") v = int(v) if v < 1: - v = 86400 * 7 + v = 86400 return v diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py index 7fa959416613..3fa592d8953a 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py @@ -37,9 +37,10 @@ def test_explicit(self): parse_idle_timeout_seconds({"idle_timeout_seconds": 30}), 30 ) - def test_rejects_zero(self): - with self.assertRaises(ValueError): - parse_idle_timeout_seconds({"idle_timeout_seconds": 0}) + def test_zero_timeout(self): + self.assertEqual( + parse_idle_timeout_seconds({"idle_timeout_seconds": 0}), 86400 + ) class TestValidateTransferConfig(unittest.TestCase): diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py index 0cfbfc40ee9e..2730c8ed16ca 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py @@ -30,9 +30,9 @@ class TestTransferIdleExpiry(ImageServerTestCase): def test_transfer_expires_after_idle(self): """No HTTP activity after registration: transfer is unregistered after idle_timeout_seconds.""" - _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=2) + _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=15) try: - time.sleep(3.5) + time.sleep(30) with self.assertRaises(urllib.error.HTTPError) as ctx: http_options(url) self.assertEqual(ctx.exception.code, 404) @@ -43,14 +43,14 @@ def test_transfer_expires_after_idle(self): def test_http_activity_resets_idle_deadline(self): """Completing a request resets the idle timer; transfer stays past a single interval.""" - _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=2) + _tid, url, _path, cleanup = make_file_transfer(idle_timeout_seconds=15) try: http_options(url) - time.sleep(1.2) + time.sleep(10) http_options(url) - time.sleep(1.2) + time.sleep(10) http_options(url) - time.sleep(1.2) + time.sleep(10) resp = http_options(url) self.assertEqual(resp.status, 200) finally: From 76793f0fa71651d0cf74ed5938669af2d40274b6 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:09:43 +0530 Subject: [PATCH 093/129] enable TLS by default and add listen address to agent.properties --- agent/conf/agent.properties | 12 ++++++----- .../agent/properties/AgentProperties.java | 15 ++++---------- .../resource/LibvirtComputingResource.java | 19 +++++------------- ...virtCreateImageTransferCommandWrapper.java | 20 ++++++++++++------- ...rtFinalizeImageTransferCommandWrapper.java | 8 ++++---- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index f2fcfd83eb13..7a74c908135d 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -78,11 +78,13 @@ zone=default # Generated with "uuidgen". local.storage.uuid= -# Enable TLS for image server transfers. -# When enabled, certificate and key paths must both be configured. -# image.server.tls.enabled=false -# image.server.tls.cert.file=/etc/cloudstack/agent/cloud.crt -# image.server.tls.key.file=/etc/cloudstack/agent/cloud.key +# Enable TLS for image server transfers. The keys are read from: +# cert file = /etc/cloudstack/agent/cloud.crt +# key file = /etc/cloudstack/agent/cloud.key +image.server.tls.enabled=true + +# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network. +#image.server.listen.address= # Location for KVM virtual router scripts. # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/". diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 22a25eaa6d89..ec60b5416055 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -126,23 +126,16 @@ public class AgentProperties{ /** * Enables TLS on the KVM image server transfer endpoint.
* Data type: Boolean.
- * Default value: false - */ - public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", false); - - /** - * PEM certificate file used by the KVM image server when TLS is enabled.
- * Data type: String.
- * Default value: null + * Default value: true */ - public static final Property IMAGE_SERVER_TLS_CERT_FILE = new Property<>("image.server.tls.cert.file", null, String.class); + public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true); /** - * PEM private key file used by the KVM image server when TLS is enabled.
+ * The IP address that the KVM image server listens on.
* Data type: String.
* Default value: null */ - public static final Property IMAGE_SERVER_TLS_KEY_FILE = new Property<>("image.server.tls.key.file", null, String.class); + public static final Property IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class); /** * Directory where Qemu sockets are placed.
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 675c9cde2667..08d84bb8d6a9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -383,6 +383,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; public static final int IMAGE_SERVER_DEFAULT_PORT = 54322; + public static final String IMAGE_SERVER_SYSTEMD_UNIT_NAME = "cloudstack-image-server"; protected int qcow2DeltaMergeTimeout; @@ -399,8 +400,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String nasBackupPath; private String imageServerPath; private boolean imageServerTlsEnabled = false; - private String imageServerTlsCertFile; - private String imageServerTlsKeyFile; + private String imageServerListenAddress; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -823,12 +823,8 @@ public boolean isImageServerTlsEnabled() { return imageServerTlsEnabled; } - public String getImageServerTlsCertFile() { - return imageServerTlsCertFile; - } - - public String getImageServerTlsKeyFile() { - return imageServerTlsKeyFile; + public String getImageServerListenAddress() { + return imageServerListenAddress; } public String getOvsPvlanDhcpHostPath() { @@ -1050,12 +1046,7 @@ public boolean configure(final String name, final Map params) th cachePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_CACHE_LOCATION); imageServerTlsEnabled = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_ENABLED); - imageServerTlsCertFile = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_CERT_FILE); - imageServerTlsKeyFile = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_KEY_FILE); - - if (imageServerTlsEnabled && (StringUtils.isBlank(imageServerTlsCertFile) || StringUtils.isBlank(imageServerTlsKeyFile))) { - throw new ConfigurationException("image server TLS is enabled but image.server.tls.cert.file or image.server.tls.key.file is missing"); - } + imageServerListenAddress = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_LISTEN_ADDRESS); params.put("domr.scripts.dir", domrScriptsDir); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 01fd11524bc7..7cf05da9b211 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -40,6 +40,9 @@ public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); + private static final String IMAGE_SERVER_TLS_CERT_FILE = "/etc/cloudstack/agent/cloud.crt"; + private static final String IMAGE_SERVER_TLS_KEY_FILE = "/etc/cloudstack/agent/cloud.key"; + private void resetService(String unitName) { Script resetScript = new Script("/bin/bash", logger); resetScript.add("-c"); @@ -51,13 +54,12 @@ private static String shellQuote(String value) { return "'" + value.replace("'", "'\\''") + "'"; } - private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputingResource resource) { + private boolean startImageServerIfNotRunning(int imageServerPort, String listenAddress, LibvirtComputingResource resource) { final String imageServerPackageDir = resource.getImageServerPath(); final String imageServerParentDir = new File(imageServerPackageDir).getParent(); final String imageServerModuleName = new File(imageServerPackageDir).getName(); - final String listenAddress = "0.0.0.0"; final boolean tlsEnabled = resource.isImageServerTlsEnabled(); - String unitName = "cloudstack-image-server"; + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); @@ -75,8 +77,8 @@ private boolean startImageServerIfNotRunning(int imageServerPort, LibvirtComputi if (tlsEnabled) { systemdRunCmd.append(" --tls-enabled"); - systemdRunCmd.append(" --tls-cert-file ").append(shellQuote(resource.getImageServerTlsCertFile())); - systemdRunCmd.append(" --tls-key-file ").append(shellQuote(resource.getImageServerTlsKeyFile())); + systemdRunCmd.append(" --tls-cert-file ").append(IMAGE_SERVER_TLS_CERT_FILE); + systemdRunCmd.append(" --tls-key-file ").append(IMAGE_SERVER_TLS_KEY_FILE); } Script startScript = new Script("/bin/bash", logger); @@ -157,7 +159,11 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r } final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; - if (!startImageServerIfNotRunning(imageServerPort, resource)) { + String listenAddress = resource.getImageServerListenAddress(); + if (StringUtils.isBlank(listenAddress)) { + listenAddress = resource.getPrivateIp(); + } + if (!startImageServerIfNotRunning(imageServerPort, listenAddress, resource)) { return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); } @@ -166,7 +172,7 @@ public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource r } final String transferScheme = resource.isImageServerTlsEnabled() ? "https" : "http"; - final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, resource.getPrivateIp(), imageServerPort, transferId); + final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, listenAddress, imageServerPort, transferId); return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java index 6c95720fda7d..3d9f6563d5eb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -39,8 +39,8 @@ private void resetService(String unitName) { resetScript.execute(); } - private boolean stopImageServer(int imageServerPort) { - String unitName = "cloudstack-image-server"; + private boolean stopImageServer(int imageServerPort, LibvirtComputingResource resource) { + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; Script checkScript = new Script("/bin/bash", logger); checkScript.add("-c"); @@ -88,12 +88,12 @@ public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); if (activeTransfers < 0) { logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); - stopImageServer(imageServerPort); + stopImageServer(imageServerPort, resource); return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); } if (activeTransfers == 0) { - stopImageServer(imageServerPort); + stopImageServer(imageServerPort, resource); } return new Answer(cmd, true, "Image transfer finalized."); From ac25dc9fb9ea77be9902d47c478c0ad620aa4114 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 6 Apr 2026 15:28:03 +0530 Subject: [PATCH 094/129] remove unused code Signed-off-by: Abhishek Kumar --- .../services/PkiResourceRouteHandler.java | 87 +------------------ 1 file changed, 3 insertions(+), 84 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java index 24c63e085dfe..e3373d5edf58 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java @@ -21,23 +21,12 @@ import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.Base64; -import java.util.Enumeration; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.ca.CAManager; -import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlServlet; import org.apache.cloudstack.veeam.utils.Negotiation; @@ -52,7 +41,6 @@ public class PkiResourceRouteHandler extends ManagerBase implements RouteHandler private static final String FORMAT_KEY = "format"; private static final String FORMAT_VALUE = "X509-PEM-CA"; private static final Charset OUTPUT_CHARSET = StandardCharsets.ISO_8859_1; - private static final boolean USE_CA_CERTS = true; @Inject CAManager caManager; @@ -89,8 +77,8 @@ protected void handleGet(HttpServletRequest req, HttpServletResponse resp, return; } - byte[] pemBytes = USE_CA_CERTS ? returnCACertificate() : returnMSCertificate(); - if (pemBytes == null || pemBytes.length == 0) { + byte[] pemBytes = returnCACertificate(); + if (pemBytes.length == 0) { resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "No certificate data available"); return; } @@ -104,7 +92,7 @@ protected void handleGet(HttpServletRequest req, HttpServletResponse resp, try (OutputStream os = resp.getOutputStream()) { os.write(pemBytes); } - } catch (IOException | KeyStoreException | CertificateException | NoSuchAlgorithmException e) { + } catch (IOException e) { String msg = "Failed to retrieve server CA certificate"; logger.error(msg, e); resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg); @@ -115,73 +103,4 @@ private byte[] returnCACertificate() throws IOException { String tlsCaCert = caManager.getCaCertificate(null); return tlsCaCert.getBytes(OUTPUT_CHARSET); } - - // ToDo: To be removed - private static byte[] returnMSCertificate() throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { - final String keystorePath = ServerPropertiesUtil.getKeystoreFile(); - final String keystorePassword = ServerPropertiesUtil.getKeystorePassword(); - - Path path = Path.of(keystorePath); - if (keystorePath.isBlank() || !Files.exists(path)) { - return null; - } - - final X509Certificate caCert = - extractCaFromKeystore(path, keystorePassword); - - // DER encoding → browser downloads as .cer (oVirt behavior) - String base64 = Base64.getMimeEncoder(64, new byte[]{'\n'}) - .encodeToString(caCert.getEncoded()); - String cert = "-----BEGIN CERTIFICATE-----\n" - + base64 - + "\n-----END CERTIFICATE-----\n"; - return cert.getBytes(OUTPUT_CHARSET); - } - - private static X509Certificate extractCaFromKeystore(Path ksPath, String ksPassword) - throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { - - final String path = ksPath.toString().toLowerCase(); - final String storeType = - (path.endsWith(".p12") || path.endsWith(".pfx")) - ? "PKCS12" - : KeyStore.getDefaultType(); - - KeyStore ks = KeyStore.getInstance(storeType); - try (var in = Files.newInputStream(ksPath)) { - ks.load(in, ksPassword != null ? ksPassword.toCharArray() : new char[0]); - } - - // Prefer HTTPS keypair alias (one with a chain) - String alias = null; - Enumeration aliases = ks.aliases(); - while (aliases.hasMoreElements()) { - String a = aliases.nextElement(); - Certificate[] chain = ks.getCertificateChain(a); - if (chain != null && chain.length > 0) { - alias = a; - break; - } - } - - if (alias == null && ks.aliases().hasMoreElements()) { - alias = ks.aliases().nextElement(); - } - - if (alias == null) { - throw new IllegalStateException("No certificate aliases in keystore"); - } - - Certificate[] chain = ks.getCertificateChain(alias); - Certificate cert = - (chain != null && chain.length > 0) - ? chain[chain.length - 1] // root-most - : ks.getCertificate(alias); - - if (!(cert instanceof X509Certificate)) { - throw new IllegalStateException("Certificate is not X509"); - } - - return (X509Certificate) cert; - } } From b52daa2be57ec1b80928067ed71fe596edaf70d7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 6 Apr 2026 15:29:16 +0530 Subject: [PATCH 095/129] changes for access checks Signed-off-by: Abhishek Kumar --- .../com/cloud/network/dao/NetworkDao.java | 3 +- .../com/cloud/network/dao/NetworkDaoImpl.java | 18 +- .../com/cloud/tags/dao/ResourceTagDao.java | 6 +- .../cloud/tags/dao/ResourceTagsDaoImpl.java | 24 +- .../backup/dao/ImageTransferDao.java | 2 + .../backup/dao/ImageTransferDaoImpl.java | 20 + .../cloudstack/veeam/adapter/ApiAccess.java | 31 + .../veeam/adapter/ApiAccessInterceptor.java | 68 + .../veeam/adapter/ServerAdapter.java | 1124 +++++++++-------- .../veeam/api/DataCentersRouteHandler.java | 3 + .../veeam/api/DisksRouteHandler.java | 22 +- .../veeam/api/ImageTransfersRouteHandler.java | 12 +- .../veeam/api/JobsRouteHandler.java | 11 +- .../veeam/api/NetworksRouteHandler.java | 13 +- .../veeam/api/TagsRouteHandler.java | 13 +- .../cloudstack/veeam/api/VmsRouteHandler.java | 13 +- .../veeam/api/VnicProfilesRouteHandler.java | 13 +- .../spring-veeam-control-service-context.xml | 14 +- .../api/query/dao/StoragePoolJoinDao.java | 3 +- .../api/query/dao/StoragePoolJoinDaoImpl.java | 7 +- .../cloud/api/query/dao/UserVmJoinDao.java | 3 +- .../api/query/dao/UserVmJoinDaoImpl.java | 24 +- .../cloud/api/query/dao/VolumeJoinDao.java | 3 +- .../api/query/dao/VolumeJoinDaoImpl.java | 18 +- .../com/cloud/api/query/vo/VolumeJoinVO.java | 5 + .../cloud/projects/ProjectManagerImpl.java | 2 +- .../com/cloud/vpc/dao/MockNetworkDaoImpl.java | 3 +- 27 files changed, 880 insertions(+), 598 deletions(-) create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccess.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptor.java diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index 243a9906486e..57b98335a280 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -101,7 +101,8 @@ public interface NetworkDao extends GenericDao, StateDao listByZoneAndTrafficType(long zoneId, TrafficType trafficType); - List listByTrafficType(TrafficType trafficType, Filter filter); + List listByTrafficTypeAndOwners(final TrafficType trafficType, List accountIds, + List domainIds, Filter filter); void setCheckForGc(long networkId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 218c447e3bc0..926e293bc2fc 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.Network; @@ -646,9 +647,22 @@ public List listByZoneAndTrafficType(final long zoneId, final Traffic } @Override - public List listByTrafficType(final TrafficType trafficType, Filter filter) { - final SearchCriteria sc = AllFieldsSearch.create(); + public List listByTrafficTypeAndOwners(final TrafficType trafficType, List accountIds, + List domainIds, Filter filter) { + SearchBuilder sb = createSearchBuilder(); + sb.and("trafficType", sb.entity().getTrafficType(), Op.EQ); + sb.and().op("account", sb.entity().getAccountId(), Op.IN); + sb.or("domain", sb.entity().getDomainId(), Op.IN); + sb.cp(); + sb.done(); + final SearchCriteria sc = sb.create(); sc.setParameters("trafficType", trafficType); + if (CollectionUtils.isNotEmpty(accountIds)) { + sc.setParameters("account", accountIds.toArray()); + } + if (CollectionUtils.isNotEmpty(domainIds)) { + sc.setParameters("domain", domainIds); + } return listBy(sc, filter); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index ccb6fea2059c..3b946eba9622 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -20,12 +20,13 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.ResourceTagResponse; + import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import org.apache.cloudstack.api.response.ResourceTagResponse; public interface ResourceTagDao extends GenericDao { @@ -62,5 +63,6 @@ public interface ResourceTagDao extends GenericDao { List listByResourceUuid(String resourceUuid); - List listByResourceType(ResourceObjectType resourceType, Filter filter); + List listByResourceTypeAndOwners(ResourceObjectType resourceType, List accountIds, + List domainIds, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index 091078f46289..47556018de4c 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -16,13 +16,14 @@ // under the License. package com.cloud.tags.dao; -import java.util.List; -import java.util.Set; -import java.util.Map; import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.server.ResourceTag; @@ -123,9 +124,22 @@ public List listByResourceUuid(String resourceUuid) { } @Override - public List listByResourceType(ResourceObjectType resourceType, Filter filter) { - SearchCriteria sc = AllFieldsSearch.create(); + public List listByResourceTypeAndOwners(ResourceObjectType resourceType, List accountIds, + List domainIds, Filter filter) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + sb.done(); + final SearchCriteria sc = sb.create();; sc.setParameters("resourceType", resourceType); + if (CollectionUtils.isNotEmpty(accountIds)) { + sc.setParameters("account", accountIds.toArray()); + } + if (CollectionUtils.isNotEmpty(domainIds)) { + sc.setParameters("domain", domainIds); + } return listBy(sc, filter); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java index e71dffb22d56..fab28dbc3421 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.ImageTransferVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface ImageTransferDao extends GenericDao { @@ -30,4 +31,5 @@ public interface ImageTransferDao extends GenericDao { ImageTransferVO findByVolume(Long volumeId); ImageTransferVO findUnfinishedByVolume(Long volumeId); List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); + List listByOwners(List accountIds, List domainIds, Filter filter); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 95741fa054d1..85dd174c129b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -23,8 +23,10 @@ import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -102,4 +104,22 @@ public List listByPhaseAndDirection(ImageTransfer.Phase phase, sc.setParameters("direction", direction); return listBy(sc); } + + @Override + public List listByOwners(List accountIds, List domainIds, Filter filter) { + SearchBuilder sb = createSearchBuilder(); + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + sb.done(); + final SearchCriteria sc = sb.create(); + if (CollectionUtils.isNotEmpty(accountIds)) { + sc.setParameters("account", accountIds.toArray()); + } + if (CollectionUtils.isNotEmpty(domainIds)) { + sc.setParameters("domain", domainIds); + } + + return listBy(sc, filter); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccess.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccess.java new file mode 100644 index 000000000000..4bb6de06e47e --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccess.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.adapter; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.cloudstack.api.BaseCmd; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface ApiAccess { + Class command(); +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptor.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptor.java new file mode 100644 index 000000000000..b0cd0cd33781 --- /dev/null +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptor.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.veeam.adapter; + +import java.lang.reflect.Method; + +import javax.inject.Inject; + +import org.aopalliance.intercept.MethodInterceptor; +import org.aopalliance.intercept.MethodInvocation; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.Pair; + +public class ApiAccessInterceptor implements MethodInterceptor { + @Inject + AccountManager accountManager; + + @Override + public Object invoke(MethodInvocation invocation) throws Throwable { + Method m = invocation.getMethod(); + Object target = invocation.getThis(); + if (target == null) { + return invocation.proceed(); + } + + ApiAccess access = m.getAnnotation(ApiAccess.class); + if (access == null) { + m = target.getClass().getMethod(m.getName(), m.getParameterTypes()); + access = m.getAnnotation(ApiAccess.class); + } + if (access == null) { + return invocation.proceed(); + } + + ServerAdapter adapter = (ServerAdapter) target; + Pair serviceUserAccount = adapter.getServiceAccount(); + String apiName = BaseCmd.getCommandNameByClass(access.command()); + + accountManager.checkApiAccess(serviceUserAccount.second(), apiName); + + CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + try { + return invocation.proceed(); + } finally { + CallContext.unregister(); + } + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index bc59d50a43a7..bc3d1aeada27 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -28,6 +28,7 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -41,31 +42,47 @@ import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiServerService; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd; +import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; +import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; +import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; +import org.apache.cloudstack.api.command.user.job.ListAsyncJobsCmd; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; +import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; import org.apache.cloudstack.api.command.user.vm.StopVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; import org.apache.cloudstack.api.command.user.vmsnapshot.CreateVMSnapshotCmd; import org.apache.cloudstack.api.command.user.vmsnapshot.DeleteVMSnapshotCmd; +import org.apache.cloudstack.api.command.user.vmsnapshot.ListVMSnapshotCmd; import org.apache.cloudstack.api.command.user.vmsnapshot.RevertToVMSnapshotCmd; import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DestroyVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.backup.BackupVO; @@ -139,6 +156,8 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.Domain; +import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -152,10 +171,11 @@ import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; import com.cloud.projects.Project; -import com.cloud.projects.ProjectService; +import com.cloud.projects.ProjectManager; import com.cloud.server.ResourceTag; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; @@ -167,6 +187,7 @@ import com.cloud.tags.dao.ResourceTagDao; import com.cloud.user.Account; import com.cloud.user.AccountService; +import com.cloud.user.DomainService; import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserDataVO; @@ -211,6 +232,11 @@ public class ServerAdapter extends ManagerBase { ResizeVolumeCmd.class, ListNetworksCmd.class ); + private static final List SUPPORTED_STORAGE_TYPES = Arrays.asList( + Storage.StoragePoolType.Filesystem, + Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.SharedMountPoint + ); public static final String WORKER_VM_GUEST_CPU_MODE = "host-passthrough"; @Inject @@ -304,7 +330,7 @@ public class ServerAdapter extends ManagerBase { NetworkModel networkModel; @Inject - ProjectService projectService; + ProjectManager projectManager; @Inject AffinityGroupDao affinityGroupDao; @@ -312,6 +338,12 @@ public class ServerAdapter extends ManagerBase { @Inject UserDataDao userDataDao; + @Inject + DomainService domainService; + + @Inject + DomainDao domainDao; + protected static Tag getDummyTagByName(String name) { Tag tag = new Tag(); String id = UUID.nameUUIDFromBytes(String.format("veeam:%s", name.toLowerCase()).getBytes()).toString(); @@ -346,7 +378,7 @@ protected Role createServiceAccountRole() { return role; } - public Role getServiceAccountRole() { + protected Role getServiceAccountRole() { List roles = roleService.findRolesByName(SERVICE_ACCOUNT_ROLE_NAME); if (CollectionUtils.isNotEmpty(roles)) { Role role = roles.get(0); @@ -437,135 +469,15 @@ protected void waitForJobCompletion(AsyncJobJoinVO job) { waitForJobCompletion(job.getId()); } - protected void validateServiceAccountAdminAccess() { - Pair serviceAccount = getServiceAccount(); - if (!accountService.isAdmin(serviceAccount.second().getId())) { - throw new InvalidParameterValueException("Service account does not have access"); - } - } - - @Override - public boolean start() { - getServiceAccount(); - return true; - } - - public List listAllDataCenters(Long offset, Long limit) { - Filter filter = new Filter(DataCenterJoinVO.class, "id", true, offset, limit); - final List clusters = dataCenterJoinDao.listAll(filter); - return DataCenterJoinVOToDataCenterConverter.toDCList(clusters); - } - - public DataCenter getDataCenter(String uuid) { - final DataCenterJoinVO vo = dataCenterJoinDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); - } - return DataCenterJoinVOToDataCenterConverter.toDataCenter(vo); - } - - public List listStorageDomainsByDcId(final String uuid, final Long offset, final Long limit) { - final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(uuid); - if (dataCenterVO == null) { - throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); - } - validateServiceAccountAdminAccess(); - Filter filter = new Filter(StoragePoolJoinVO.class, "id", true, offset, limit); - List storagePoolVOS = storagePoolJoinDao.listByZoneAndProvider(dataCenterVO.getId(), filter); - return StoreVOToStorageDomainConverter.toStorageDomainListFromPools(storagePoolVOS); - } - - public List listNetworksByDcId(final String uuid, final Long offset, final Long limit) { - final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); - if (dataCenterVO == null) { - throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); - } - validateServiceAccountAdminAccess(); - Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); - List networks = networkDao.listByZoneAndTrafficType(dataCenterVO.getId(), Networks.TrafficType.Guest, filter); - return NetworkVOToNetworkConverter.toNetworkList(networks, (dcId) -> dataCenterVO); - } - - public List listAllClusters(Long offset, Long limit) { - validateServiceAccountAdminAccess(); - Filter filter = new Filter(ClusterVO.class, "id", true, offset, limit); - final List clusters = clusterDao.listByHypervisorType(Hypervisor.HypervisorType.KVM, filter); - return ClusterVOToClusterConverter.toClusterList(clusters, this::getZoneById); + protected ApiServerService.AsyncCmdResult processAsyncCmdWithContext(BaseAsyncCmd cmd, Map params) + throws Exception { + final CallContext ctx = CallContext.current(); + final long callerUserId = ctx.getCallingUserId(); + final Account caller = ctx.getCallingAccount(); + return apiServerService.processAsyncCmd(cmd, params, ctx, callerUserId, caller); } - public Cluster getCluster(String uuid) { - validateServiceAccountAdminAccess(); - final ClusterVO vo = clusterDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); - } - return ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); - } - - public List listAllHosts(Long offset, Long limit) { - validateServiceAccountAdminAccess(); - Filter filter = new Filter(HostJoinVO.class, "id", true, offset, limit); - final List hosts = hostJoinDao.listRoutingHostsByHypervisor(Hypervisor.HypervisorType.KVM, filter); - return HostJoinVOToHostConverter.toHostList(hosts); - } - - public Host getHost(String uuid) { - validateServiceAccountAdminAccess(); - final HostJoinVO vo = hostJoinDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); - } - return HostJoinVOToHostConverter.toHost(vo); - } - - public List listAllNetworks(Long offset, Long limit) { - Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); - final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest, filter); - return NetworkVOToNetworkConverter.toNetworkList(networks, this::getZoneById); - } - - public Network getNetwork(String uuid) { - final NetworkVO vo = networkDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Network with ID " + uuid + " not found"); - } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); - return NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); - } - - public List listAllVnicProfiles(Long offset, Long limit) { - Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); - final List networks = networkDao.listByTrafficType(Networks.TrafficType.Guest, filter); - return NetworkVOToVnicProfileConverter.toVnicProfileList(networks, this::getZoneById); - } - - public VnicProfile getVnicProfile(String uuid) { - final NetworkVO vo = networkDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Nic profile with ID " + uuid + " not found"); - } - return NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); - } - - public List listAllInstances(Long offset, Long limit) { - Filter filter = new Filter(UserVmJoinVO.class, "id", true, offset, limit); - List vms = userVmJoinDao.listByHypervisorType(Hypervisor.HypervisorType.KVM, filter); - return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); - } - - public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, boolean allContent) { - UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); - } - return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, - this::getDetailsByInstanceId, - includeDisks ? this::listDiskAttachmentsByInstanceId : null, - includeNics ? this::listNicsByInstance : null, - allContent); - } - - Account getOwnerForInstanceCreation(Vm request) { + protected Account getOwnerForInstanceCreation(Vm request) { if (!VeeamControlService.InstanceRestoreAssignOwner.value()) { return null; } @@ -581,14 +493,14 @@ Account getOwnerForInstanceCreation(Vm request) { return account; } - Ternary getOwnerDetailsForInstanceCreation(Account account) { + protected Ternary getOwnerDetailsForInstanceCreation(Account account) { if (account == null) { return new Ternary<>(null, null, null); } String accountName = account.getAccountName(); Long projectId = null; if (Account.Type.PROJECT.equals(account.getType())) { - Project project = projectService.findByProjectAccountId(account.getId()); + Project project = projectManager.findByProjectAccountId(account.getId()); if (project == null) { logger.warn("Project for {} not found, unable to determine owner for VM creation request", account); return new Ternary<>(null, null, null); @@ -599,79 +511,46 @@ Ternary getOwnerDetailsForInstanceCreation(Account account) return new Ternary<>(account.getDomainId(), accountName, projectId); } - public Vm createInstance(Vm request) { - if (request == null) { - throw new InvalidParameterValueException("Request disk data is empty"); + protected Pair, String> getResourceOwnerFilters() { + final Account caller = CallContext.current().getCallingAccount(); + final Account.Type type = caller.getType(); + if (Account.Type.ADMIN.equals(type)) { + return new Pair<>(null, null); } - OvfXmlUtil.updateFromConfiguration(request); - String name = request.getName(); - if (StringUtils.isBlank(name)) { - throw new InvalidParameterValueException("Invalid name specified for the VM"); + List permittedAccountIds = null; + String domainPath = null; + if (Account.Type.DOMAIN_ADMIN.equals(type) || Account.Type.NORMAL.equals(type)) { + permittedAccountIds = projectManager.listPermittedProjectAccounts(caller.getId()); + permittedAccountIds.add(caller.getId()); } - String displayName = name; - name = name.replace("_", "-"); - Long zoneId = null; - Long clusterId = null; - if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { - ClusterVO clusterVO = clusterDao.findByUuid(request.getCluster().getId()); - if (clusterVO != null) { - zoneId = clusterVO.getDataCenterId(); - clusterId = clusterVO.getId(); + if (Account.Type.DOMAIN_ADMIN.equals(type)) { + Domain domain = domainService.getDomain(caller.getDomainId()); + if (domain == null) { + throw new InvalidParameterValueException("Invalid service account specified"); } + domainPath = domain.getPath(); } - if (zoneId == null) { - throw new InvalidParameterValueException("Failed to determine datacenter for VM creation request"); - } - DataCenterVO zone = dataCenterDao.findById(zoneId); - if (zone == null) { - throw new InvalidParameterValueException("DataCenter could not be determined for the request"); - } - Integer cpu = null; - try { - cpu = Integer.valueOf(request.getCpu().getTopology().getSockets()); - } catch (Exception ignored) { - } - if (cpu == null) { - throw new InvalidParameterValueException("CPU topology sockets must be specified"); - } - Long memory = null; - try { - memory = Long.valueOf(request.getMemory()); - } catch (Exception ignored) { - } - if (memory == null) { - throw new InvalidParameterValueException("Memory must be specified"); - } - int memoryMB = (int)(memory / (1024L * 1024L)); - String userdata = null; - if (request.getInitialization() != null) { - userdata = request.getInitialization().getCustomScript(); - } - Pair bootOptions = Vm.Bios.retrieveBootOptions(request.getBios()); - Account owner = getOwnerForInstanceCreation(request); - Ternary ownerDetails = getOwnerDetailsForInstanceCreation(owner); - String serviceOfferingUuid = null; - if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { - serviceOfferingUuid = request.getCpuProfile().getId(); - } - String templateUuid = null; - if (request.getTemplate() != null && StringUtils.isNotEmpty(request.getTemplate().getId())) { - templateUuid = request.getTemplate().getId(); + if (Account.Type.PROJECT.equals(type)) { + Project project = projectManager.findByProjectAccountId(caller.getId()); + if (project == null) { + throw new InvalidParameterValueException("Invalid service account specified"); + } + permittedAccountIds = new ArrayList<>(); + permittedAccountIds.add(caller.getId()); } - Pair serviceUserAccount = getServiceAccount(); - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - return createInstance(zone, clusterId, owner, ownerDetails.first(), ownerDetails.second(), - ownerDetails.third(), name, displayName, serviceOfferingUuid, cpu, memoryMB, templateUuid, - userdata, bootOptions.first(), bootOptions.second(), request.getAffinityGroupId(), - request.getUserDataId(), request.getDetails()); - } finally { - CallContext.unregister(); + return new Pair<>(permittedAccountIds, domainPath); + } + + protected Pair, List> getResourceOwnerFiltersWithDomainIds() { + Pair, String> filters = getResourceOwnerFilters(); + if (StringUtils.isNotBlank(filters.second())) { + return new Pair<>(filters.first(), domainDao.getDomainChildrenIds(filters.second())); } + return new Pair<>(filters.first(), null); } protected ServiceOfferingVO getServiceOfferingFromRequest(com.cloud.dc.DataCenter zone, Account account, - String uuid, int cpu, int memory) { + String uuid, int cpu, int memory) { if (StringUtils.isBlank(uuid)) { return null; } @@ -711,7 +590,7 @@ protected ServiceOfferingVO getServiceOfferingFromRequest(com.cloud.dc.DataCente } protected ServiceOffering getServiceOfferingIdForVmCreation(com.cloud.dc.DataCenter zone, Account account, - String serviceOfferingUuid, int cpu, int memory) { + String serviceOfferingUuid, int cpu, int memory) { ServiceOfferingVO offering = getServiceOfferingFromRequest(zone, account, serviceOfferingUuid, cpu, memory); if (offering != null) { return offering; @@ -747,9 +626,9 @@ protected VMTemplateVO getTemplateForInstanceCreation(String templateUuid) { } protected Vm createInstance(com.cloud.dc.DataCenter zone, Long clusterId, Account owner, Long domainId, - String accountName, Long projectId, String name, String displayName, String serviceOfferingUuid, - int cpu, int memory, String templateUuid, String userdata, ApiConstants.BootType bootType, - ApiConstants.BootMode bootMode, String affinityGroupId, String userDataId, Map details) { + String accountName, Long projectId, String name, String displayName, String serviceOfferingUuid, + int cpu, int memory, String templateUuid, String userdata, ApiConstants.BootType bootType, + ApiConstants.BootMode bootMode, String affinityGroupId, String userDataId, Map details) { Account account = owner != null ? owner : CallContext.current().getCallingAccount(); ServiceOffering serviceOffering = getServiceOfferingIdForVmCreation(zone, account, serviceOfferingUuid, cpu, memory); @@ -786,84 +665,414 @@ protected Vm createInstance(com.cloud.dc.DataCenter zone, Long clusterId, Accoun if (template != null) { cmd.setTemplateId(template.getId()); } - if (StringUtils.isNotBlank(affinityGroupId)) { - AffinityGroupVO group = affinityGroupDao.findByUuid(affinityGroupId); - if (group == null) { - logger.warn("Failed to find affinity group with ID {} specified in Instance creation request, " + - "skipping affinity group assignment", affinityGroupId); - } else { - cmd.setAffinityGroupIds(List.of(group.getId())); + if (StringUtils.isNotBlank(affinityGroupId)) { + AffinityGroupVO group = affinityGroupDao.findByUuid(affinityGroupId); + if (group == null) { + logger.warn("Failed to find affinity group with ID {} specified in Instance creation request, " + + "skipping affinity group assignment", affinityGroupId); + } else { + cmd.setAffinityGroupIds(List.of(group.getId())); + } + } + if (StringUtils.isNotBlank(userDataId)) { + UserDataVO userData = userDataDao.findByUuid(userDataId); + if (userData == null) { + logger.warn("Failed to find userdata with ID {} specified in Instance creation request, " + + "skipping userdata assignment", userDataId); + } else { + cmd.setUserDataId(userData.getId()); + } + } + cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); + Map instanceDetails = getDetailsForInstanceCreation(userdata, serviceOffering, details); + if (MapUtils.isNotEmpty(instanceDetails)) { + Map> map = new HashMap<>(); + map.put(0, instanceDetails); + cmd.setDetails(map); + } + cmd.setBlankInstance(true); + try { + UserVm vm = userVmManager.createVirtualMachine(cmd); + vm = userVmManager.finalizeCreateVirtualMachine(vm.getId()); + UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, + this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); + } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { + throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); + } + } + + @NotNull + protected static Map getDetailsForInstanceCreation(String userdata, ServiceOffering serviceOffering, + Map existingDetails) { + Map details = new HashMap<>(); + List detailsTobeSkipped = List.of( + ApiConstants.BootType.BIOS.toString(), + ApiConstants.BootType.UEFI.toString()); + if (MapUtils.isNotEmpty(existingDetails)) { + for (Map.Entry entry : existingDetails.entrySet()) { + if (detailsTobeSkipped.contains(entry.getKey())) { + continue; + } + details.put(entry.getKey(), entry.getValue()); + } + } + if (StringUtils.isNotEmpty(userdata)) { + // Assumption: Only worker VM will have userdata and it needs CPU mode + details.put(VmDetailConstants.GUEST_CPU_MODE, WORKER_VM_GUEST_CPU_MODE); + } + if (serviceOffering.isCustomized()) { + details.put(VmDetailConstants.CPU_NUMBER, String.valueOf(serviceOffering.getCpu())); + details.put(VmDetailConstants.MEMORY, String.valueOf(serviceOffering.getRamSize())); + if (serviceOffering.getSpeed() == null && !details.containsKey(VmDetailConstants.CPU_SPEED)) { + details.put(VmDetailConstants.CPU_SPEED, String.valueOf(1000)); + } + } + return details; + } + + protected static long getProvisionedSizeInGb(String sizeStr) { + long provisionedSizeInGb; + try { + provisionedSizeInGb = Long.parseLong(sizeStr); + } catch (NumberFormatException ex) { + throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); + } + if (provisionedSizeInGb <= 0) { + throw new InvalidParameterValueException("Provisioned size must be greater than zero"); + } + // round-up provisionedSizeInGb to the next whole GB + long GB = 1024L * 1024L * 1024L; + provisionedSizeInGb = Math.max(1L, (provisionedSizeInGb + GB - 1) / GB); + return provisionedSizeInGb; + } + + protected Long getVolumePhysicalSize(VolumeJoinVO vo) { + return volumeApiService.getVolumePhysicalSize(vo.getFormat(), vo.getPath(), vo.getChainInfo()); + } + + @NotNull + protected Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { + Volume volume; + try { + volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, + null, name, sizeInGb, null, null, null, null); + } catch (ResourceAllocationException e) { + throw new CloudRuntimeException(e.getMessage(), e); + } + if (volume == null) { + throw new CloudRuntimeException("Failed to create volume"); + } + volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); + if (initialSize != null) { + volumeDetailsDao.addDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE, String.valueOf(initialSize), true); + } + + // Implementation for creating a Disk resource + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId()), this::getVolumePhysicalSize); + } + + protected List listNicsByInstance(final long instanceId, final String instanceUuid) { + List nics = nicDao.listByVmId(instanceId); + return NicVOToNicConverter.toNicList(nics, instanceUuid, this::getNetworkById); + } + + protected List listNicsByInstance(final UserVmJoinVO vo) { + return listNicsByInstance(vo.getId(), vo.getUuid()); + } + + protected boolean accountCannotAccessNetwork(NetworkVO networkVO, long accountId) { + Account account = accountService.getActiveAccountById(accountId); + try { + networkModel.checkNetworkPermissions(account, networkVO); + return false; + } catch (CloudRuntimeException e) { + logger.debug("{} cannot access {}: {}", account, networkVO, e.getMessage()); + } + return true; + } + + protected void assignVmToAccount(UserVmVO vmVO, long accountId) { + Account account = accountService.getActiveAccountById(accountId); + if (account == null) { + throw new InvalidParameterValueException("Account with ID " + accountId + " not found"); + } + try { + AssignVMCmd cmd = new AssignVMCmd(); + ComponentContext.inject(cmd); + cmd.setVirtualMachineId(vmVO.getId()); + cmd.setDomainId(account.getDomainId()); + if (Account.Type.PROJECT.equals(account.getType())) { + Project project = projectManager.findByProjectAccountId(account.getId()); + if (project == null) { + throw new InvalidParameterValueException("Project for " + account + " not found"); + } + cmd.setProjectId(project.getId()); + } else { + cmd.setAccountName(account.getAccountName()); + } + cmd.setSkipNetwork(true); + userVmManager.moveVmToUser(cmd); + } catch (ResourceAllocationException | CloudRuntimeException | ResourceUnavailableException | + InsufficientCapacityException e) { + logger.error("Failed to assign {} to {}: {}", vmVO, account, e.getMessage(), e); + } + } + + protected ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) { + org.apache.cloudstack.backup.ImageTransfer imageTransfer = + kvmBackupExportService.createImageTransfer(volumeId, backupId, direction, format); + ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); + return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, + this::getVolumeById); + } + + protected DataCenterJoinVO getZoneById(Long zoneId) { + if (zoneId == null) { + return null; + } + return dataCenterJoinDao.findById(zoneId); + } + + protected HostJoinVO getHostById(Long hostId) { + if (hostId == null) { + return null; + } + return hostJoinDao.findById(hostId); + } + + protected VolumeJoinVO getVolumeById(Long volumeId) { + if (volumeId == null) { + return null; + } + return volumeJoinDao.findById(volumeId); + } + + protected NetworkVO getNetworkById(Long networkId) { + if (networkId == null) { + return null; + } + return networkDao.findById(networkId); + } + + protected Map getDetailsByInstanceId(Long instanceId) { + return vmInstanceDetailsDao.listDetailsKeyPairs(instanceId, true); + } + + @Override + public boolean start() { + getServiceAccount(); + return true; + } + + @ApiAccess(command = ListZonesCmd.class) + public List listAllDataCenters(Long offset, Long limit) { + Filter filter = new Filter(DataCenterJoinVO.class, "id", true, offset, limit); + final List clusters = dataCenterJoinDao.listAll(filter); + return DataCenterJoinVOToDataCenterConverter.toDCList(clusters); + } + + @ApiAccess(command = ListZonesCmd.class) + public DataCenter getDataCenter(String uuid) { + final DataCenterJoinVO vo = dataCenterJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + return DataCenterJoinVOToDataCenterConverter.toDataCenter(vo); + } + + @ApiAccess(command = ListStoragePoolsCmd.class) + public List listStorageDomainsByDcId(final String uuid, final Long offset, final Long limit) { + final DataCenterVO dataCenterVO = dataCenterDao.findByUuid(uuid); + if (dataCenterVO == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + Filter filter = new Filter(StoragePoolJoinVO.class, "id", true, offset, limit); + List storagePoolVOS = storagePoolJoinDao.listByZoneAndType(dataCenterVO.getId(), + SUPPORTED_STORAGE_TYPES, filter); + return StoreVOToStorageDomainConverter.toStorageDomainListFromPools(storagePoolVOS); + } + + @ApiAccess(command = ListNetworksCmd.class) + public List listNetworksByDcId(final String uuid, final Long offset, final Long limit) { + final DataCenterJoinVO dataCenterVO = dataCenterJoinDao.findByUuid(uuid); + if (dataCenterVO == null) { + throw new InvalidParameterValueException("DataCenter with ID " + uuid + " not found"); + } + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + List networks = networkDao.listByZoneAndTrafficType(dataCenterVO.getId(), Networks.TrafficType.Guest, filter); + return NetworkVOToNetworkConverter.toNetworkList(networks, (dcId) -> dataCenterVO); + } + + @ApiAccess(command = ListClustersCmd.class) + public List listAllClusters(Long offset, Long limit) { + Filter filter = new Filter(ClusterVO.class, "id", true, offset, limit); + final List clusters = clusterDao.listByHypervisorType(Hypervisor.HypervisorType.KVM, filter); + return ClusterVOToClusterConverter.toClusterList(clusters, this::getZoneById); + } + + @ApiAccess(command = ListClustersCmd.class) + public Cluster getCluster(String uuid) { + final ClusterVO vo = clusterDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Cluster with ID " + uuid + " not found"); + } + return ClusterVOToClusterConverter.toCluster(vo, this::getZoneById); + } + + @ApiAccess(command = ListHostsCmd.class) + public List listAllHosts(Long offset, Long limit) { + Filter filter = new Filter(HostJoinVO.class, "id", true, offset, limit); + final List hosts = hostJoinDao.listRoutingHostsByHypervisor(Hypervisor.HypervisorType.KVM, filter); + return HostJoinVOToHostConverter.toHostList(hosts); + } + + @ApiAccess(command = ListHostsCmd.class) + public Host getHost(String uuid) { + final HostJoinVO vo = hostJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Host with ID " + uuid + " not found"); + } + return HostJoinVOToHostConverter.toHost(vo); + } + + @ApiAccess(command = ListNetworksCmd.class) + public List listAllNetworks(Long offset, Long limit) { + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); + final List networks = networkDao.listByTrafficTypeAndOwners(Networks.TrafficType.Guest, + ownerDetails.first(), ownerDetails.second(), filter); + return NetworkVOToNetworkConverter.toNetworkList(networks, this::getZoneById); + } + + @ApiAccess(command = ListNetworksCmd.class) + public Network getNetwork(String uuid) { + final NetworkVO vo = networkDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Network with ID " + uuid + " not found"); + } + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); + return NetworkVOToNetworkConverter.toNetwork(vo, this::getZoneById); + } + + @ApiAccess(command = ListNetworksCmd.class) + public List listAllVnicProfiles(Long offset, Long limit) { + Filter filter = new Filter(NetworkVO.class, "id", true, offset, limit); + Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); + final List networks = networkDao.listByTrafficTypeAndOwners(Networks.TrafficType.Guest, + ownerDetails.first(), ownerDetails.second(), filter); + return NetworkVOToVnicProfileConverter.toVnicProfileList(networks, this::getZoneById); + } + + @ApiAccess(command = ListNetworksCmd.class) + public VnicProfile getVnicProfile(String uuid) { + final NetworkVO vo = networkDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Nic profile with ID " + uuid + " not found"); + } + return NetworkVOToVnicProfileConverter.toVnicProfile(vo, this::getZoneById); + } + + @ApiAccess(command = ListVMsCmd.class) + public List listAllInstances(Long offset, Long limit) { + Filter filter = new Filter(UserVmJoinVO.class, "id", true, offset, limit); + Pair, String> ownerDetails = getResourceOwnerFilters(); + List vms = userVmJoinDao.listByHypervisorTypeAndOwners(Hypervisor.HypervisorType.KVM, + ownerDetails.first(), ownerDetails.second(), filter); + return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); + } + + @ApiAccess(command = ListVMsCmd.class) + public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, boolean allContent) { + UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); + } + return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, + this::getDetailsByInstanceId, + includeDisks ? this::listDiskAttachmentsByInstanceId : null, + includeNics ? this::listNicsByInstance : null, + allContent); + } + + @ApiAccess(command = DeployVMCmd.class) + public Vm createInstance(Vm request) { + if (request == null) { + throw new InvalidParameterValueException("Request disk data is empty"); + } + OvfXmlUtil.updateFromConfiguration(request); + String name = request.getName(); + if (StringUtils.isBlank(name)) { + throw new InvalidParameterValueException("Invalid name specified for the VM"); + } + String displayName = name; + name = name.replace("_", "-"); + Long zoneId = null; + Long clusterId = null; + if (request.getCluster() != null && StringUtils.isNotEmpty(request.getCluster().getId())) { + ClusterVO clusterVO = clusterDao.findByUuid(request.getCluster().getId()); + if (clusterVO != null) { + zoneId = clusterVO.getDataCenterId(); + clusterId = clusterVO.getId(); } } - if (StringUtils.isNotBlank(userDataId)) { - UserDataVO userData = userDataDao.findByUuid(userDataId); - if (userData == null) { - logger.warn("Failed to find userdata with ID {} specified in Instance creation request, " + - "skipping userdata assignment", userDataId); - } else { - cmd.setUserDataId(userData.getId()); - } + if (zoneId == null) { + throw new InvalidParameterValueException("Failed to determine datacenter for VM creation request"); } - cmd.setHypervisor(Hypervisor.HypervisorType.KVM.name()); - Map instanceDetails = getDetailsForInstanceCreation(userdata, serviceOffering, details); - if (MapUtils.isNotEmpty(instanceDetails)) { - Map> map = new HashMap<>(); - map.put(0, instanceDetails); - cmd.setDetails(map); + DataCenterVO zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("DataCenter could not be determined for the request"); } - cmd.setBlankInstance(true); + Integer cpu = null; try { - UserVm vm = userVmManager.createVirtualMachine(cmd); - vm = userVmManager.finalizeCreateVirtualMachine(vm.getId()); - UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); - return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, - this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); - } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { - throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); + cpu = Integer.valueOf(request.getCpu().getTopology().getSockets()); + } catch (Exception ignored) { } - } - - @NotNull - private static Map getDetailsForInstanceCreation(String userdata, ServiceOffering serviceOffering, - Map existingDetails) { - Map details = new HashMap<>(); - List detailsTobeSkipped = List.of( - ApiConstants.BootType.BIOS.toString(), - ApiConstants.BootType.UEFI.toString()); - if (MapUtils.isNotEmpty(existingDetails)) { - for (Map.Entry entry : existingDetails.entrySet()) { - if (detailsTobeSkipped.contains(entry.getKey())) { - continue; - } - details.put(entry.getKey(), entry.getValue()); - } + if (cpu == null) { + throw new InvalidParameterValueException("CPU topology sockets must be specified"); } - if (StringUtils.isNotEmpty(userdata)) { - // Assumption: Only worker VM will have userdata and it needs CPU mode - details.put(VmDetailConstants.GUEST_CPU_MODE, WORKER_VM_GUEST_CPU_MODE); + Long memory = null; + try { + memory = Long.valueOf(request.getMemory()); + } catch (Exception ignored) { } - if (serviceOffering.isCustomized()) { - details.put(VmDetailConstants.CPU_NUMBER, String.valueOf(serviceOffering.getCpu())); - details.put(VmDetailConstants.MEMORY, String.valueOf(serviceOffering.getRamSize())); - if (serviceOffering.getSpeed() == null && !details.containsKey(VmDetailConstants.CPU_SPEED)) { - details.put(VmDetailConstants.CPU_SPEED, String.valueOf(1000)); - } + if (memory == null) { + throw new InvalidParameterValueException("Memory must be specified"); } - return details; + int memoryMB = (int)(memory / (1024L * 1024L)); + String userdata = null; + if (request.getInitialization() != null) { + userdata = request.getInitialization().getCustomScript(); + } + Pair bootOptions = Vm.Bios.retrieveBootOptions(request.getBios()); + Account owner = getOwnerForInstanceCreation(request); + Ternary ownerDetails = getOwnerDetailsForInstanceCreation(owner); + String serviceOfferingUuid = null; + if (request.getCpuProfile() != null && StringUtils.isNotEmpty(request.getCpuProfile().getId())) { + serviceOfferingUuid = request.getCpuProfile().getId(); + } + String templateUuid = null; + if (request.getTemplate() != null && StringUtils.isNotEmpty(request.getTemplate().getId())) { + templateUuid = request.getTemplate().getId(); + } + return createInstance(zone, clusterId, owner, ownerDetails.first(), ownerDetails.second(), + ownerDetails.third(), name, displayName, serviceOfferingUuid, cpu, memoryMB, templateUuid, + userdata, bootOptions.first(), bootOptions.second(), request.getAffinityGroupId(), + request.getUserDataId(), request.getDetails()); } + @ApiAccess(command = UpdateVMCmd.class) public Vm updateInstance(String uuid, Vm request) { logger.warn("Received request to update VM with ID {}. No action, returning existing VM data.", uuid); return getInstance(uuid, false, false, false); } + @ApiAccess(command = DestroyVMCmd.class) public VmAction deleteInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { DestroyVMCmd cmd = new DestroyVMCmd(); cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); @@ -871,9 +1080,7 @@ public VmAction deleteInstance(String uuid, boolean async) { Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); params.put(ApiConstants.EXPUNGE, Boolean.TRUE.toString()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for VM deletion"); @@ -884,27 +1091,22 @@ public VmAction deleteInstance(String uuid, boolean async) { return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete VM: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = StartVMCmd.class) public VmAction startInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StartVMCmd cmd = new StartVMCmd(); cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for VM start"); @@ -915,18 +1117,15 @@ public VmAction startInstance(String uuid, boolean async) { return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to start VM: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = StopVMCmd.class) public VmAction stopInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); @@ -934,9 +1133,7 @@ public VmAction stopInstance(String uuid, boolean async) { Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); params.put(ApiConstants.FORCED, Boolean.TRUE.toString()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for VM stop"); @@ -947,18 +1144,15 @@ public VmAction stopInstance(String uuid, boolean async) { return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to stop VM: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = StopVMCmd.class) public VmAction shutdownInstance(String uuid, boolean async) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { StopVMCmd cmd = new StopVMCmd(); cmd.setHttpMethod(BaseCmd.HTTPMethod.POST.name()); @@ -966,9 +1160,7 @@ public VmAction shutdownInstance(String uuid, boolean async) { Map params = new HashMap<>(); params.put(ApiConstants.ID, vo.getUuid()); params.put(ApiConstants.FORCED, Boolean.FALSE.toString()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for VM shutdown"); @@ -979,27 +1171,25 @@ public VmAction shutdownInstance(String uuid, boolean async) { return AsyncJobJoinVOToJobConverter.toVmAction(jobVo, userVmJoinDao.findById(vo.getId())); } catch (Exception e) { throw new CloudRuntimeException("Failed to shutdown VM: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } - protected Long getVolumePhysicalSize(VolumeJoinVO vo) { - return volumeApiService.getVolumePhysicalSize(vo.getFormat(), vo.getPath(), vo.getChainInfo()); - } - + @ApiAccess(command = ListVolumesCmd.class) public List listAllDisks(Long offset, Long limit) { Filter filter = new Filter(VolumeJoinVO.class, "id", true, offset, limit); - List kvmVolumes = volumeJoinDao.listByHypervisor(Hypervisor.HypervisorType.KVM, filter); + Pair, String> ownerDetails = getResourceOwnerFilters(); + List kvmVolumes = volumeJoinDao.listByHypervisorTypeAndOwners(Hypervisor.HypervisorType.KVM, + ownerDetails.first(), ownerDetails.second(), filter); return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes, this::getVolumePhysicalSize); } + @ApiAccess(command = ListVolumesCmd.class) public Disk getDisk(String uuid) { VolumeVO vo = volumeDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findByUuid(uuid), this::getVolumePhysicalSize); } @@ -1011,26 +1201,27 @@ public Disk reduceDisk(String uuid) { throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); } + @ApiAccess(command = ListVolumesCmd.class) protected List listDiskAttachmentsByInstanceId(final long instanceId) { List kvmVolumes = volumeJoinDao.listByInstanceId(instanceId); return VolumeJoinVOToDiskConverter.toDiskAttachmentList(kvmVolumes, this::getVolumePhysicalSize); } + @ApiAccess(command = ListVolumesCmd.class) public List listDiskAttachmentsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return listDiskAttachmentsByInstanceId(vo.getId()); } - protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId, Pair serviceUserAccount) { + protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId) { Account account = accountService.getActiveAccountById(accountId); if (account == null) { throw new InvalidParameterValueException("Account with ID " + accountId + " not found"); } - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { AssignVolumeCmd cmd = new AssignVolumeCmd(); ComponentContext.inject(cmd); @@ -1038,7 +1229,7 @@ protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId, Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vmVo); if (request == null || request.getDisk() == null || StringUtils.isEmpty(request.getDisk().getId())) { throw new InvalidParameterValueException("Request disk data is empty"); } @@ -1071,30 +1261,27 @@ public DiskAttachment attachInstanceDisk(final String vmUuid, final DiskAttachme if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vmVo); if (vmVo.getAccountId() != volumeVO.getAccountId()) { if (VeeamControlService.InstanceRestoreAssignOwner.value()) { - assignVolumeToAccount(volumeVO, vmVo.getAccountId(), serviceUserAccount); + assignVolumeToAccount(volumeVO, vmVo.getAccountId()); } else { throw new PermissionDeniedException("Disk with ID " + request.getDisk().getId() + " belongs to a different account and cannot be attached to the VM"); } } - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - Long deviceId = null; - List volumes = volumeDao.findUsableVolumesForInstance(vmVo.getId()); - if (CollectionUtils.isEmpty(volumes)) { - deviceId = 0L; - } - Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), deviceId, false); - VolumeJoinVO attachedVolumeVO = volumeJoinDao.findById(volume.getId()); - return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO, this::getVolumePhysicalSize); - } finally { - CallContext.unregister(); + Long deviceId = null; + List volumes = volumeDao.findUsableVolumesForInstance(vmVo.getId()); + if (CollectionUtils.isEmpty(volumes)) { + deviceId = 0L; } + Volume volume = volumeApiService.attachVolumeToVM(vmVo.getId(), volumeVO.getId(), deviceId, false); + VolumeJoinVO attachedVolumeVO = volumeJoinDao.findById(volume.getId()); + return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO, this::getVolumePhysicalSize); } + @ApiAccess(command = DestroyVolumeCmd.class) public void deleteDisk(String uuid) { VolumeVO vo = volumeDao.findByUuid(uuid); if (vo == null) { @@ -1103,6 +1290,7 @@ public void deleteDisk(String uuid) { volumeApiService.deleteVolume(vo.getId(), accountService.getSystemAccount()); } + @ApiAccess(command = CreateVolumeCmd.class) public Disk createDisk(Disk request) { if (request == null) { throw new InvalidParameterValueException("Request disk data is empty"); @@ -1131,127 +1319,36 @@ public Disk createDisk(Disk request) { initialSize = Long.parseLong(request.getInitialSize()); } catch (NumberFormatException ignored) {} } - Pair serviceUserAccount = getServiceAccount(); - Account serviceAccount = serviceUserAccount.second(); + Account caller = CallContext.current().getCallingAccount(); DataCenterVO zone = dataCenterDao.findById(pool.getDataCenterId()); if (zone == null || !Grouping.AllocationState.Enabled.equals(zone.getAllocationState())) { throw new InvalidParameterValueException("Datacenter for the specified storage domain is not found or not active"); } - Long diskOfferingId = volumeApiService.getCustomDiskOfferingIdForVolumeUpload(serviceAccount, zone); + Long diskOfferingId = volumeApiService.getCustomDiskOfferingIdForVolumeUpload(caller, zone); if (diskOfferingId == null) { throw new CloudRuntimeException("Failed to find custom offering for disk" + zone.getName()); } - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - return createDisk(serviceAccount, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); - } finally { - CallContext.unregister(); - } - } - - private static long getProvisionedSizeInGb(String sizeStr) { - long provisionedSizeInGb; - try { - provisionedSizeInGb = Long.parseLong(sizeStr); - } catch (NumberFormatException ex) { - throw new InvalidParameterValueException("Invalid provisioned size: " + sizeStr); - } - if (provisionedSizeInGb <= 0) { - throw new InvalidParameterValueException("Provisioned size must be greater than zero"); - } - // round-up provisionedSizeInGb to the next whole GB - long GB = 1024L * 1024L * 1024L; - provisionedSizeInGb = Math.max(1L, (provisionedSizeInGb + GB - 1) / GB); - return provisionedSizeInGb; - } - - @NotNull - private Disk createDisk(Account serviceAccount, StoragePoolVO pool, String name, Long diskOfferingId, long sizeInGb, Long initialSize) { - Volume volume; - try { - volume = volumeApiService.allocVolume(serviceAccount.getId(), pool.getDataCenterId(), diskOfferingId, null, - null, name, sizeInGb, null, null, null, null); - } catch (ResourceAllocationException e) { - throw new CloudRuntimeException(e.getMessage(), e); - } - if (volume == null) { - throw new CloudRuntimeException("Failed to create volume"); - } - volume = volumeApiService.createVolume(volume.getId(), null, null, pool.getId(), true); - if (initialSize != null) { - volumeDetailsDao.addDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE, String.valueOf(initialSize), true); - } - - // Implementation for creating a Disk resource - return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findById(volume.getId()), this::getVolumePhysicalSize); - } - - protected List listNicsByInstance(final long instanceId, final String instanceUuid) { - List nics = nicDao.listByVmId(instanceId); - return NicVOToNicConverter.toNicList(nics, instanceUuid, this::getNetworkById); - } - - protected List listNicsByInstance(final UserVmJoinVO vo) { - return listNicsByInstance(vo.getId(), vo.getUuid()); + return createDisk(caller, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); } + @ApiAccess(command = ListNicsCmd.class) public List listNicsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return listNicsByInstance(vo.getId(), vo.getUuid()); } - protected boolean accountCannotAccessNetwork(NetworkVO networkVO, long accountId) { - Account account = accountService.getActiveAccountById(accountId); - try { - networkModel.checkNetworkPermissions(account, networkVO); - return false; - } catch (CloudRuntimeException e) { - logger.debug("{} cannot access {}: {}", account, networkVO, e.getMessage()); - } - return true; - } - - protected void assignVmToAccount(UserVmVO vmVO, long accountId, Pair serviceUserAccount) { - Account account = accountService.getActiveAccountById(accountId); - if (account == null) { - throw new InvalidParameterValueException("Account with ID " + accountId + " not found"); - } - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - AssignVMCmd cmd = new AssignVMCmd(); - ComponentContext.inject(cmd); - cmd.setVirtualMachineId(vmVO.getId()); - cmd.setDomainId(account.getDomainId()); - if (Account.Type.PROJECT.equals(account.getType())) { - Project project = projectService.findByProjectAccountId(account.getId()); - if (project == null) { - throw new InvalidParameterValueException("Project for " + account + " not found"); - } - cmd.setProjectId(project.getId()); - } else { - cmd.setAccountName(account.getAccountName()); - } - cmd.setSkipNetwork(true); - userVmManager.moveVmToUser(cmd); - } catch (ResourceAllocationException | CloudRuntimeException | ResourceUnavailableException | - InsufficientCapacityException e) { - logger.error("Failed to assign {} to {}: {}", vmVO, account, e.getMessage(), e); - } finally { - CallContext.unregister(); - } - } - + @ApiAccess(command = AddNicToVMCmd.class) public Nic attachInstanceNic(final String vmUuid, final Nic request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vmVo); if (request == null || request.getVnicProfile() == null || StringUtils.isEmpty(request.getVnicProfile().getId())) { throw new InvalidParameterValueException("Request nic data is empty"); } @@ -1259,48 +1356,49 @@ public Nic attachInstanceNic(final String vmUuid, final Nic request) { if (networkVO == null) { throw new InvalidParameterValueException("VNic profile " + request.getVnicProfile().getId() + " not found"); } - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, networkVO); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, networkVO); if (vmVo.getAccountId() != networkVO.getAccountId() && networkVO.getAccountId() != Account.ACCOUNT_ID_SYSTEM && VeeamControlService.InstanceRestoreAssignOwner.value() && accountCannotAccessNetwork(networkVO, vmVo.getAccountId())) { - assignVmToAccount(vmVo, networkVO.getAccountId(), serviceUserAccount); + assignVmToAccount(vmVo, networkVO.getAccountId()); } - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - AddNicToVMCmd cmd = new AddNicToVMCmd(); - ComponentContext.inject(cmd); - cmd.setVmId(vmVo.getId()); - cmd.setNetworkId(networkVO.getId()); - if (request.getMac() != null && StringUtils.isNotBlank(request.getMac().getAddress())) { - cmd.setMacAddress(request.getMac().getAddress()); - } - userVmManager.addNicToVirtualMachine(cmd); - NicVO nic = nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkVO.getId(), vmVo.getId()); - if (nic == null) { - throw new CloudRuntimeException("Failed to attach NIC to VM"); - } - return NicVOToNicConverter.toNic(nic, vmUuid, this::getNetworkById); - } finally { - CallContext.unregister(); + AddNicToVMCmd cmd = new AddNicToVMCmd(); + ComponentContext.inject(cmd); + cmd.setVmId(vmVo.getId()); + cmd.setNetworkId(networkVO.getId()); + if (request.getMac() != null && StringUtils.isNotBlank(request.getMac().getAddress())) { + cmd.setMacAddress(request.getMac().getAddress()); + } + userVmManager.addNicToVirtualMachine(cmd); + NicVO nic = nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkVO.getId(), vmVo.getId()); + if (nic == null) { + throw new CloudRuntimeException("Failed to attach NIC to VM"); } + return NicVOToNicConverter.toNic(nic, vmUuid, this::getNetworkById); } + @ApiAccess(command = ListImageTransfersCmd.class) public List listAllImageTransfers(Long offset, Long limit) { Filter filter = new Filter(ImageTransferVO.class, "id", true, offset, limit); - List imageTransfers = imageTransferDao.listAll(filter); + Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); + List imageTransfers = imageTransferDao.listByOwners(ownerDetails.first(), + ownerDetails.second(), filter); return ImageTransferVOToImageTransferConverter.toImageTransferList(imageTransfers, this::getHostById, this::getVolumeById); } + @ApiAccess(command = ListImageTransfersCmd.class) public ImageTransfer getImageTransfer(String uuid) { ImageTransferVO vo = imageTransferDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return ImageTransferVOToImageTransferConverter.toImageTransfer(vo, this::getHostById, this::getVolumeById); } + @ApiAccess(command = CreateImageTransferCmd.class) public ImageTransfer createImageTransfer(ImageTransfer request) { if (request == null) { throw new InvalidParameterValueException("Request image transfer data is empty"); @@ -1312,8 +1410,8 @@ public ImageTransfer createImageTransfer(ImageTransfer request) { if (volumeVO == null) { throw new InvalidParameterValueException("Disk with ID " + request.getDisk().getId() + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), null, false, volumeVO); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, + volumeVO); Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); if (direction == null) { throw new InvalidParameterValueException("Invalid or missing direction"); @@ -1327,88 +1425,47 @@ public ImageTransfer createImageTransfer(ImageTransfer request) { } backupId = backupVO.getId(); } - return createImageTransfer(backupId, volumeVO.getId(), direction, format, serviceUserAccount); + return createImageTransfer(backupId, volumeVO.getId(), direction, format); } + @ApiAccess(command = FinalizeImageTransferCmd.class) public boolean cancelImageTransfer(String uuid) { ImageTransferVO vo = imageTransferDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, vo); return kvmBackupExportService.cancelImageTransfer(vo.getId()); } + @ApiAccess(command = FinalizeImageTransferCmd.class) public boolean finalizeImageTransfer(String uuid) { ImageTransferVO vo = imageTransferDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Image transfer with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, vo); return kvmBackupExportService.finalizeImageTransfer(vo.getId()); } - private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format, - Pair serviceUserAccount) { - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); - try { - org.apache.cloudstack.backup.ImageTransfer imageTransfer = - kvmBackupExportService.createImageTransfer(volumeId, backupId, direction, format); - ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId()); - return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById); - } finally { - CallContext.unregister(); - } - } - - protected DataCenterJoinVO getZoneById(Long zoneId) { - if (zoneId == null) { - return null; - } - return dataCenterJoinDao.findById(zoneId); - } - - private HostJoinVO getHostById(Long hostId) { - if (hostId == null) { - return null; - } - return hostJoinDao.findById(hostId); - } - - private VolumeJoinVO getVolumeById(Long volumeId) { - if (volumeId == null) { - return null; - } - return volumeJoinDao.findById(volumeId); - } - - protected NetworkVO getNetworkById(Long networkId) { - if (networkId == null) { - return null; - } - return networkDao.findById(networkId); - } - - protected Map getDetailsByInstanceId(Long instanceId) { - return vmInstanceDetailsDao.listDetailsKeyPairs(instanceId, true); - } - + @ApiAccess(command = ListAsyncJobsCmd.class) public List listPendingJobs() { - Pair serviceUserAccount = getServiceAccount(); - List jobIds = asyncJobDao.listPendingJobIdsForAccount(serviceUserAccount.second().getId()); + List jobIds = asyncJobDao.listPendingJobIdsForAccount(CallContext.current().getCallingAccountId()); List jobJoinVOs = asyncJobJoinDao.listByIds(jobIds); return AsyncJobJoinVOToJobConverter.toJobList(jobJoinVOs); } + @ApiAccess(command = ListAsyncJobsCmd.class) public Job getJob(String uuid) { final AsyncJobJoinVO vo = asyncJobJoinDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Job with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return AsyncJobJoinVOToJobConverter.toJob(vo); } + @ApiAccess(command = ListVMSnapshotCmd.class) public List listSnapshotsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { @@ -1418,14 +1475,14 @@ public List listSnapshotsByInstanceUuid(final String uuid) { return VmSnapshotVOToSnapshotConverter.toSnapshotList(snapshots, vo.getUuid()); } + @ApiAccess(command = CreateVMSnapshotCmd.class) public Snapshot createInstanceSnapshot(final String vmUuid, final Snapshot request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vmVo); try { CreateVMSnapshotCmd cmd = new CreateVMSnapshotCmd(); ComponentContext.inject(cmd); @@ -1433,9 +1490,7 @@ public Snapshot createInstanceSnapshot(final String vmUuid, final Snapshot reque params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); params.put(ApiConstants.VM_SNAPSHOT_DESCRIPTION, request.getDescription()); params.put(ApiConstants.VM_SNAPSHOT_MEMORY, String.valueOf(Boolean.parseBoolean(request.getPersistMemorystate()))); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); if (result.objectId == null) { throw new CloudRuntimeException("No snapshot ID returned"); } @@ -1446,17 +1501,16 @@ public Snapshot createInstanceSnapshot(final String vmUuid, final Snapshot reque return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vmVo.getUuid()); } catch (Exception e) { throw new CloudRuntimeException("Failed to create snapshot: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = ListVMSnapshotCmd.class) public Snapshot getSnapshot(String uuid) { VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); UserVmVO vm = userVmDao.findById(vo.getVmId()); return VmSnapshotVOToSnapshotConverter.toSnapshot(vo, vm.getUuid()); } @@ -1467,17 +1521,14 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vo); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vo); try { DeleteVMSnapshotCmd cmd = new DeleteVMSnapshotCmd(); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for snapshot deletion"); @@ -1488,29 +1539,25 @@ public ResourceAction deleteSnapshot(String uuid, boolean async) { action = AsyncJobJoinVOToJobConverter.toAction(jobVo); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete snapshot: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } return action; } + @ApiAccess(command = RevertToVMSnapshotCmd.class) public ResourceAction revertInstanceToSnapshot(String uuid, boolean async) { ResourceAction action = null; VMSnapshotVO vo = vmSnapshotDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("Snapshot with ID " + uuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vo); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vo); try { RevertToVMSnapshotCmd cmd = new RevertToVMSnapshotCmd(); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.VM_SNAPSHOT_ID, vo.getUuid()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, serviceUserAccount.first().getId(), - serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); AsyncJobJoinVO jobVo = asyncJobJoinDao.findById(result.jobId); if (jobVo == null) { throw new CloudRuntimeException("Failed to find job for snapshot revert"); @@ -1521,12 +1568,11 @@ public ResourceAction revertInstanceToSnapshot(String uuid, boolean async) { action = AsyncJobJoinVOToJobConverter.toAction(jobVo); } catch (Exception e) { throw new CloudRuntimeException("Failed to revert to snapshot: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } return action; } + @ApiAccess(command = ListBackupsCmd.class) public List listBackupsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { @@ -1536,14 +1582,27 @@ public List listBackupsByInstanceUuid(final String uuid) { return BackupVOToBackupConverter.toBackupList(backups, id -> vo, this::getHostById); } + protected void validateInstanceStorage(UserVmVO vm) { + List volumes = volumeDao.findUsableVolumesForInstance(vm.getId()); + List storageIds = volumes.stream().map(VolumeVO::getPoolId).distinct().collect(Collectors.toList()); + List pools = primaryDataStoreDao.listByIds(storageIds); + pools.stream().filter(p -> !SUPPORTED_STORAGE_TYPES.contains(p.getPoolType())) + .findAny().ifPresent(p -> { + throw new InvalidParameterValueException("VM is using storage pool " + p.getName() + + " of type " + p.getPoolType() + + " which is not supported for backup operations"); + }); + } + + @ApiAccess(command = StartBackupCmd.class) public Backup createInstanceBackup(final String vmUuid, final Backup request) { UserVmVO vmVo = userVmDao.findByUuid(vmUuid); if (vmVo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, vmVo); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, vmVo); + validateInstanceStorage(vmVo); try { StartBackupCmd cmd = new StartBackupCmd(); ComponentContext.inject(cmd); @@ -1551,8 +1610,7 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { params.put(ApiConstants.VIRTUAL_MACHINE_ID, vmVo.getUuid()); params.put(ApiConstants.NAME, request.getName()); params.put(ApiConstants.DESCRIPTION, request.getDescription()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, vmVo.getUserId(), serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); if (result == null || result.objectId == null) { throw new CloudRuntimeException("Unexpected backup ID returned"); } @@ -1563,26 +1621,27 @@ public Backup createInstanceBackup(final String vmUuid, final Backup request) { return BackupVOToBackupConverter.toBackup(vo, id -> vmVo, this::getHostById, this::getBackupDisks); } catch (Exception e) { throw new CloudRuntimeException("Failed to create backup: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = ListBackupsCmd.class) public Backup getBackup(String uuid) { BackupVO vo = backupDao.findByUuidIncludingRemoved(uuid); if (vo == null) { throw new InvalidParameterValueException("Backup with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); return BackupVOToBackupConverter.toBackup(vo, id -> userVmDao.findById(id), this::getHostById, this::getBackupDisks); } + @ApiAccess(command = ListBackupsCmd.class) public List listDisksByBackupUuid(final String uuid) { throw new InvalidParameterValueException("List Backup Disks with ID " + uuid + " not implemented"); // This won't be feasible with current structure } + @ApiAccess(command = FinalizeBackupCmd.class) public Backup finalizeBackup(final String vmUuid, final String backupUuid) { UserVmVO vm = userVmDao.findByUuid(vmUuid); if (vm == null) { @@ -1592,17 +1651,15 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { if (backup == null) { throw new InvalidParameterValueException("Backup with ID " + backupUuid + " not found"); } - Pair serviceUserAccount = getServiceAccount(); - accountService.checkAccess(serviceUserAccount.second(), SecurityChecker.AccessType.OperateEntry, false, backup); - CallContext ctx = CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, + false, backup); try { FinalizeBackupCmd cmd = new FinalizeBackupCmd(); ComponentContext.inject(cmd); Map params = new HashMap<>(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, vm.getUuid()); params.put(ApiConstants.ID, backup.getUuid()); - ApiServerService.AsyncCmdResult result = - apiServerService.processAsyncCmd(cmd, params, ctx, vm.getUserId(), serviceUserAccount.second()); + ApiServerService.AsyncCmdResult result = processAsyncCmdWithContext(cmd, params); if (result == null) { throw new CloudRuntimeException("Failed to finalize backup"); } @@ -1610,11 +1667,10 @@ public Backup finalizeBackup(final String vmUuid, final String backupUuid) { return BackupVOToBackupConverter.toBackup(backup, id -> vm, this::getHostById, this::getBackupDisks); } catch (Exception e) { throw new CloudRuntimeException("Failed to finalize backup: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = ListBackupsCmd.class) protected List getBackupDisks(final BackupVO backup) { List volumeInfos = backup.getBackedUpVolumes(); if (CollectionUtils.isEmpty(volumeInfos)) { @@ -1623,12 +1679,13 @@ protected List getBackupDisks(final BackupVO backup) { return VolumeJoinVOToDiskConverter.toDiskListFromVolumeInfos(volumeInfos); } + @ApiAccess(command = ListVmCheckpointsCmd.class) public List listCheckpointsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), null, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); Checkpoint checkpoint = UserVmVOToCheckpointConverter.toCheckpoint(vo); if (checkpoint == null) { return Collections.emptyList(); @@ -1636,18 +1693,17 @@ public List listCheckpointsByInstanceUuid(final String uuid) { return List.of(checkpoint); } + @ApiAccess(command = DeleteVmCheckpointCmd.class) public void deleteCheckpoint(String vmUuid, String checkpointId) { UserVmVO vo = userVmDao.findByUuid(vmUuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } - accountService.checkAccess(getServiceAccount().second(), SecurityChecker.AccessType.OperateEntry, false, vo); + accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, vo); if (!Objects.equals(vo.getActiveCheckpointId(), checkpointId)) { logger.warn("Checkpoint ID {} does not match active checkpoint for VM {}", checkpointId, vmUuid); return; } - Pair serviceUserAccount = getServiceAccount(); - CallContext.register(serviceUserAccount.first(), serviceUserAccount.second()); try { DeleteVmCheckpointCmd cmd = new DeleteVmCheckpointCmd(); ComponentContext.inject(cmd); @@ -1655,22 +1711,23 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { kvmBackupExportService.deleteVmCheckpoint(cmd); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); - } finally { - CallContext.unregister(); } } + @ApiAccess(command = ListTagsCmd.class) public List listAllTags(final Long offset, final Long limit) { List tags = new ArrayList<>(getDummyTags().values()); Filter filter = new Filter(ResourceTagVO.class, "id", true, offset, limit); - List vmResourceTags = resourceTagDao.listByResourceType(ResourceTag.ResourceObjectType.UserVm, - filter); + Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); + List vmResourceTags = resourceTagDao.listByResourceTypeAndOwners( + ResourceTag.ResourceObjectType.UserVm, ownerDetails.first(), ownerDetails.second(), filter); if (CollectionUtils.isNotEmpty(vmResourceTags)) { tags.addAll(ResourceTagVOToTagConverter.toTags(vmResourceTags)); } return tags; } + @ApiAccess(command = ListTagsCmd.class) public Tag getTag(String uuid) { if (BaseDto.ZERO_UUID.equals(uuid)) { return ResourceTagVOToTagConverter.getRootTag(); @@ -1678,7 +1735,8 @@ public Tag getTag(String uuid) { Tag tag = getDummyTags().get(uuid); if (tag == null) { ResourceTagVO resourceTagVO = resourceTagDao.findByUuid(uuid); - accountService.checkAccess(getServiceAccount().second(), null, false, resourceTagVO); + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, + resourceTagVO); if (resourceTagVO != null) { tag = ResourceTagVOToTagConverter.toTag(resourceTagVO); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 4ff5add7d3d8..7e68375fe567 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -39,6 +39,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; public class DataCentersRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/datacenters"; @@ -111,6 +112,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index f4cd3b6a3785..0dd316753552 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -121,10 +121,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllDisks(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("disk", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllDisks(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("disk", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, @@ -161,15 +165,7 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, protected void handlePutById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String data = RouteHandler.getRequestData(req, logger); - try { - // ToDo: do what? -// serverAdapter.deleteDisk(id); - Disk response = serverAdapter.getDisk(id); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); - } catch (InvalidParameterValueException e) { - io.badRequest(resp, e.getMessage(), outFormat); - } + throw new InvalidParameterValueException("Put Disk with ID " + id + " not implemented"); } protected void handlePostDiskCopy(final String id, final HttpServletRequest req, final HttpServletResponse resp, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index 00b473eb6a41..ef27d6353fbf 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -106,10 +106,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllImageTransfers(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("image_transfer", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllImageTransfers(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("image_transfer", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index 0cb038127697..50a7465414e6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -35,6 +35,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; public class JobsRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/jobs"; @@ -84,9 +85,13 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - final List result = serverAdapter.listPendingJobs(); - NamedList response = NamedList.of("job", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + final List result = serverAdapter.listPendingJobs(); + NamedList response = NamedList.of("job", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index 4014dc796fe6..31e5bccca7c7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -36,6 +36,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; public class NetworksRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/networks"; @@ -85,10 +86,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllNetworks(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("network", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllNetworks(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("network", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java index b571bcaa2ede..727cf72ca1a5 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java @@ -36,6 +36,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; public class TagsRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/tags"; @@ -86,10 +87,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllTags(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("tag", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllTags(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("tag", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index fdf542d64714..a3d1ca236cbf 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -231,10 +231,14 @@ protected static boolean isRequestAsync(HttpServletRequest req) { protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllInstances(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("vm", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllInstances(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("vm", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handlePost(final HttpServletRequest req, final HttpServletResponse resp, @@ -308,7 +312,6 @@ protected void handleStartVmById(final String id, final HttpServletRequest req, protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { boolean async = isRequestAsync(req); - String data = RouteHandler.getRequestData(req, logger); try { VmAction vm = serverAdapter.stopInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index 3e8aab2176fd..31fb93ddf3b7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -36,6 +36,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; public class VnicProfilesRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api/vnicprofiles"; @@ -85,10 +86,14 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { - ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllVnicProfiles(query.getOffset(), query.getLimit()); - NamedList response = NamedList.of("vnic_profile", result); - io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + try { + ListQuery query = ListQuery.fromRequest(req); + final List result = serverAdapter.listAllVnicProfiles(query.getOffset(), query.getLimit()); + NamedList response = NamedList.of("vnic_profile", result); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handleGetById(final String id, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index cbe11724648b..4d66d5248e04 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -18,8 +18,11 @@ --> + + + + + + diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java index dc19d8481933..54a98a225bcb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse; import com.cloud.api.query.vo.StoragePoolJoinVO; +import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; @@ -45,6 +46,6 @@ public interface StoragePoolJoinDao extends GenericDao List findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags); - List listByZoneAndProvider(long zoneId, Filter filter); + List listByZoneAndType(long zoneId, List types, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 35651f657941..5f4527a7c555 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; @@ -412,12 +413,16 @@ public List findStoragePoolByScopeAndRuleTags(Long datacenterId, } @Override - public List listByZoneAndProvider(long zoneId, Filter filter) { + public List listByZoneAndType(long zoneId, List types, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.and("types", sb.entity().getZoneId(), SearchCriteria.Op.IN); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("zoneId", zoneId); + if (CollectionUtils.isNotEmpty(types)) { + sc.setParameters("types", types.toArray()); + } return listBy(sc, filter); } } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java index 55d65df7ffb3..0612e9066665 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java @@ -52,5 +52,6 @@ List listByAccountServiceOfferingTemplateAndNotInState(long accoun List listLeaseInstancesExpiringInDays(int days); - List listByHypervisorType(Hypervisor.HypervisorType hypervisorType, Filter filter); + List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, List accountIds, + String domainPath, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index d243bb7a5468..94b25ccc82f5 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -17,14 +17,13 @@ package com.cloud.api.query.dao; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Collections; import java.time.LocalDate; import java.time.ZoneId; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.Date; - import java.util.HashMap; import java.util.Hashtable; import java.util.List; @@ -34,9 +33,6 @@ import javax.inject.Inject; -import com.cloud.gpu.dao.VgpuProfileDao; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.service.dao.ServiceOfferingDao; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -62,11 +58,14 @@ import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.gpu.GPU; +import com.cloud.gpu.dao.VgpuProfileDao; import com.cloud.host.ControlState; +import com.cloud.hypervisor.Hypervisor; import com.cloud.network.IpAddress; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.service.ServiceOfferingDetailsVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; import com.cloud.storage.Storage.TemplateType; @@ -96,7 +95,6 @@ import com.cloud.vm.VmStats; import com.cloud.vm.dao.NicExtraDhcpOptionDao; import com.cloud.vm.dao.NicSecondaryIpVO; - import com.cloud.vm.dao.VMInstanceDetailsDao; @Component @@ -836,12 +834,22 @@ public List listLeaseInstancesExpiringInDays(int days) { } @Override - public List listByHypervisorType(Hypervisor.HypervisorType hypervisorType, Filter filter) { + public List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, + List accountIds, String domainPath, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("hypervisorType", sb.entity().getHypervisorType(), Op.EQ); + sb.and().op("account", sb.entity().getAccountId(), Op.IN); + sb.or("domainPath", sb.entity().getDomainPath(), Op.LIKE); + sb.cp(); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("hypervisorType", hypervisorType); + if (CollectionUtils.isNotEmpty(accountIds)) { + sc.setParameters("account", accountIds.toArray()); + } + if (StringUtils.isNotBlank(domainPath)) { + sc.setParameters("domainPath", domainPath + "%"); + } return listBy(sc, filter); } } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java index c3b5859120fb..7cfdfbe78e68 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDao.java @@ -39,5 +39,6 @@ public interface VolumeJoinDao extends GenericDao { List listByInstanceId(long instanceId); - List listByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter); + List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, List accountIds, + String domainPath, Filter filter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 20b6d69c5917..4bcb8bff6879 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -21,8 +21,6 @@ import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.offering.DiskOffering; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ResponseObject.ResponseView; @@ -31,11 +29,15 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -382,14 +384,24 @@ public List listByInstanceId(long instanceId) { } @Override - public List listByHypervisor(Hypervisor.HypervisorType hypervisorType, Filter filter) { + public List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, + List accountIds, String domainPath, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("vmType", sb.entity().getVmType(), SearchCriteria.Op.EQ); sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domainPath", sb.entity().getDomainPath(), SearchCriteria.Op.LIKE); + sb.cp(); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("vmType", VirtualMachine.Type.User); sc.setParameters("hypervisorType", hypervisorType); + if (CollectionUtils.isNotEmpty(accountIds)) { + sc.setParameters("account", accountIds.toArray()); + } + if (StringUtils.isNotBlank(domainPath)) { + sc.setParameters("domainPath", domainPath + "%"); + } return search(sc, filter); } diff --git a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java index 2ae720fa8524..ba932c775beb 100644 --- a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java @@ -23,6 +23,7 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; +import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; @@ -40,6 +41,10 @@ @Table(name = "volume_view") public class VolumeJoinVO extends BaseViewWithTagInformationVO implements ControlledViewEntity { + @Id + @Column(name = "id", updatable = false, nullable = false) + private long id; + @Column(name = "uuid") private String uuid; diff --git a/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java b/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java index 95fdfa0fde89..a5ca5b1bf07d 100644 --- a/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java +++ b/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java @@ -476,7 +476,7 @@ public ProjectAccount assignAccountToProject(Project project, long accountId, Pr return _projectAccountDao.persist(projectAccountVO); } - public ProjectAccount assignUserToProject(Project project, long userId, long accountId, Role userRole, Long projectRoleId) { + public ProjectAccount assignUserToProject(Project project, long userId, long accountId, Role userRole, Long projectRoleId) { return assignAccountToProject(project, accountId, userRole, userId, projectRoleId); } diff --git a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java index 4c646b5264b6..adcbc5d01734 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java @@ -172,7 +172,8 @@ public List listByZoneAndTrafficType(final long zoneId, final Traffic } @Override - public List listByTrafficType(final TrafficType trafficType, Filter filter) { + public List listByTrafficTypeAndOwners(final TrafficType trafficType, List accountIds, + List domainIds, Filter filter) { return null; } From d6055c9ae2ae21b16592e4142d28c41f48d24f02 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 5 Apr 2026 15:44:36 +0530 Subject: [PATCH 096/129] create volume on storage refactor --- .../command/user/volume/CreateVolumeCmd.java | 5 +++- .../cloud/storage/VolumeApiServiceImpl.java | 28 +++++-------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 6371a3598abc..15926c55e873 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -113,7 +113,7 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, - description = "Storage pool ID to create the volume in.") + description = "Storage pool ID to create the volume in. Exclusive with SnapshotId parameter.") private Long storageId; ///////////////////////////////////////////////////// @@ -161,6 +161,9 @@ private Long getProjectId() { } public Long getStorageId() { + if (snapshotId != null && storageId != null) { + throw new IllegalArgumentException("StorageId parameter cannot be specified with the SnapshotId parameter."); + } return storageId; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 3b833a1c1503..1e91c300e9bb 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1048,29 +1048,15 @@ public boolean validateVolumeSizeInBytes(long size) { return true; } - private VolumeVO allocateVolumeOnStorage(Long volumeId, Long storageId) { + private VolumeVO allocateVolumeOnStorage(Long volumeId, Long storageId) throws ExecutionException, InterruptedException { DataStore destStore = dataStoreMgr.getDataStore(storageId, DataStoreRole.Primary); VolumeInfo destVolume = volFactory.getVolume(volumeId, destStore); - try { - AsyncCallFuture createVolumeFuture = volService.createVolumeAsync(destVolume, destStore); - VolumeApiResult createVolumeResult = createVolumeFuture.get(); - if (createVolumeResult.isFailed()) { - logger.debug("Failed to create dest volume {}, volume can be removed", destVolume); - destroyVolume(destVolume.getId()); - destVolume.processEvent(ObjectInDataStoreStateMachine.Event.ExpungeRequested); - destVolume.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); - _volsDao.remove(destVolume.getId()); - throw new CloudRuntimeException("Creation of a dest volume failed: " + createVolumeResult.getResult()); - } else { - destVolume = volFactory.getVolume(destVolume.getId(), destStore); - destVolume.processEvent(ObjectInDataStoreStateMachine.Event.CreateRequested); - destVolume.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); - } - } catch (Exception e) { - logger.debug("Failed to create dest volume {}", destVolume, e); - throw new CloudRuntimeException("Creation of a dest volume failed: volume needs cleanup"); + AsyncCallFuture createVolumeFuture = volService.createVolumeAsync(destVolume, destStore); + VolumeApiResult createVolumeResult = createVolumeFuture.get(); + if (createVolumeResult.isFailed()) { + throw new CloudRuntimeException("Creation of a dest volume failed: " + createVolumeResult.getResult()); } - return null; + return _volsDao.findById(destVolume.getId()); } @Override @@ -1113,7 +1099,7 @@ public VolumeVO createVolume(long volumeId, Long vmId, Long snapshotId, Long sto } } } else if (storageId != null) { - allocateVolumeOnStorage(volumeId, storageId); + volume = allocateVolumeOnStorage(volumeId, storageId); } return volume; } catch (Exception e) { From b84ff6b99a794469dc4f8165e16bfc94a032b6dc Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 7 Apr 2026 04:31:05 +0530 Subject: [PATCH 097/129] move checkpoint to vm details --- .../java/com/cloud/vm/VmDetailConstants.java | 6 ++ .../cloudstack/backup/StartBackupAnswer.java | 9 -- .../main/java/com/cloud/vm/VMInstanceVO.java | 22 ----- .../META-INF/db/schema-42210to42300.sql | 4 - .../veeam/adapter/ServerAdapter.java | 8 +- .../UserVmVOToCheckpointConverter.java | 14 +-- .../backup/KVMBackupExportServiceImpl.java | 96 +++++++++++-------- 7 files changed, 74 insertions(+), 85 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9e56bf4f17b2..33cc6da70812 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -130,4 +130,10 @@ public interface VmDetailConstants { String EXTERNAL_DETAIL_PREFIX = "External:"; String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details"; String CLOUDSTACK_VLAN = "cloudstack.vlan"; + + // KVM Checkpoints related + String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id"; + String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time"; + String LAST_CHECKPOINT_ID = "last.checkpoint.id"; + String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time"; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java index 7628fe19698f..d7cbf097df90 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -21,7 +21,6 @@ public class StartBackupAnswer extends Answer { private Long checkpointCreateTime; - private Boolean isIncremental; public StartBackupAnswer() { } @@ -42,12 +41,4 @@ public Long getCheckpointCreateTime() { public void setCheckpointCreateTime(Long checkpointCreateTime) { this.checkpointCreateTime = checkpointCreateTime; } - - public Boolean getIncremental() { - return isIncremental; - } - - public void setIncremental(Boolean incremental) { - isIncremental = incremental; - } } diff --git a/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java b/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java index 1678caaa525b..9d5e1b0ff500 100644 --- a/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java @@ -202,12 +202,6 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject getBackupVolumeList() { public void setBackupVolumes(String backupVolumes) { this.backupVolumes = backupVolumes; } - - public String getActiveCheckpointId() { - return activeCheckpointId; - } - - public void setActiveCheckpointId(String activeCheckpointId) { - this.activeCheckpointId = activeCheckpointId; - } - - public Long getActiveCheckpointCreateTime() { - return activeCheckpointCreateTime; - } - - public void setActiveCheckpointCreateTime(Long activeCheckpointCreateTime) { - this.activeCheckpointCreateTime = activeCheckpointCreateTime; - } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 90f8d1d61eb1..47b28964acdc 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -124,10 +124,6 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCH CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); --- Add checkpoint tracking fields to vm_instance table for domain recreation -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Active checkpoint id tracked for incremental backups"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'active_checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Active checkpoint creation time"'); - -- Create image_transfer table for per-disk image transfers CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index bc3d1aeada27..0593476b74cb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1686,7 +1686,10 @@ public List listCheckpointsByInstanceUuid(final String uuid) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); - Checkpoint checkpoint = UserVmVOToCheckpointConverter.toCheckpoint(vo); + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vo.getId()); + Checkpoint checkpoint = UserVmVOToCheckpointConverter.toCheckpoint( + details.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID), + details.get(VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME)); if (checkpoint == null) { return Collections.emptyList(); } @@ -1700,7 +1703,8 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { throw new InvalidParameterValueException("VM with ID " + vmUuid + " not found"); } accountService.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, vo); - if (!Objects.equals(vo.getActiveCheckpointId(), checkpointId)) { + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vo.getId()); + if (!Objects.equals(details.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID), checkpointId)) { logger.warn("Checkpoint ID {} does not match active checkpoint for VM {}", checkpointId, vmUuid); return; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java index 019bc8264c84..7f64b6b7d4aa 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java @@ -22,19 +22,19 @@ import org.apache.cloudstack.veeam.api.dto.Checkpoint; import org.apache.commons.lang3.StringUtils; -import com.cloud.vm.UserVmVO; +import com.cloud.utils.NumbersUtil; public class UserVmVOToCheckpointConverter { - public static Checkpoint toCheckpoint(final UserVmVO vm) { - if (StringUtils.isEmpty(vm.getActiveCheckpointId())) { + public static Checkpoint toCheckpoint(String checkpointId, String createTimeStr) { + if (StringUtils.isEmpty(checkpointId)) { return null; } Checkpoint checkpoint = new Checkpoint(); - checkpoint.setId(vm.getActiveCheckpointId()); - checkpoint.setName(vm.getActiveCheckpointId()); - Long createTimeSeconds = vm.getActiveCheckpointCreateTime(); - if (createTimeSeconds != null) { + checkpoint.setId(checkpointId); + checkpoint.setName(checkpointId); + long createTimeSeconds = createTimeStr != null ? NumbersUtil.parseLong(createTimeStr, 0L) : 0L; + if (createTimeSeconds > 0) { checkpoint.setCreationDate(String.valueOf(Instant.ofEpochSecond(createTimeSeconds).toEpochMilli())); } else { checkpoint.setCreationDate(String.valueOf(System.currentTimeMillis())); diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index d71f7b668480..e1b89b07e5ed 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -78,7 +78,9 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; @@ -89,6 +91,9 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject private VMInstanceDao vmInstanceDao; + @Inject + private VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject private BackupDao backupDao; @@ -164,15 +169,14 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setDate(new Date()); String toCheckpointId = "ckp-" + UUID.randomUUID().toString().substring(0, 8); - String fromCheckpointId = vm.getActiveCheckpointId(); + Map vmDetails = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); + String fromCheckpointId = vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID); backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); backup.setHostId(hostId); - // Will be changed later if incremental was done - backup.setType("FULL"); return backupDao.persist(backup); } @@ -200,11 +204,14 @@ public Backup startBackup(StartBackupCmd cmd) { long hostId = backup.getHostId(); Host host = hostDao.findById(hostId); + Map vmDetails = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); + String activeCkpCreateTimeStr = vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); + Long fromCheckpointCreateTime = activeCkpCreateTimeStr != null ? NumbersUtil.parseLong(activeCkpCreateTimeStr, 0L) : null; StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), backup.getToCheckpointId(), backup.getFromCheckpointId(), - vm.getActiveCheckpointCreateTime(), + fromCheckpointCreateTime, backup.getUuid(), diskPathUuidMap, vm.getState() == State.Stopped @@ -227,10 +234,6 @@ public Backup startBackup(StartBackupCmd cmd) { // Update backup with checkpoint creation time backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); - if (Boolean.TRUE.equals(answer.getIncremental())) { - // todo: set it in the backend - backup.setType("Incremental"); - } updateBackupState(backup, Backup.Status.ReadyForTransfer); return backup; } @@ -240,6 +243,24 @@ protected void updateBackupState(BackupVO backup, Backup.Status newStatus) { backupDao.update(backup.getId(), backup); } + private void updateVmCheckpoints(Long vmId, BackupVO backup) { + Map vmDetails = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); + String oldCheckpointId = vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID); + String oldCreateTimeStr = vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); + if (oldCheckpointId != null && oldCreateTimeStr != null) { + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.LAST_CHECKPOINT_ID, oldCheckpointId, false); + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.LAST_CHECKPOINT_CREATE_TIME, oldCreateTimeStr, false); + } + String newCheckpointId = backup.getToCheckpointId(); + Long newCreateTime = backup.getCheckpointCreateTime(); + if (newCheckpointId != null && newCreateTime != null) { + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_ID, backup.getToCheckpointId(), false); + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME, String.valueOf(newCreateTime), false); + } else { + logger.error("New checkpoint details are missing for backup {} and vm {}", backup.getId(), vmId); + } + } + @Override public Backup finalizeBackup(FinalizeBackupCmd cmd) { Long vmId = cmd.getVmId(); @@ -277,29 +298,18 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { try { answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { - updateBackupState(backup, Backup.Status.Failed); + removeFailedBackup(backup); throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { - updateBackupState(backup, Backup.Status.Failed); + removeFailedBackup(backup); throw new CloudRuntimeException("Failed to stop backup: " + answer.getDetails()); } } - // Update VM checkpoint tracking - String oldCheckpointId = vm.getActiveCheckpointId(); - vm.setActiveCheckpointId(backup.getToCheckpointId()); - vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); - vmInstanceDao.update(vmId, vm); + updateVmCheckpoints(vmId, backup); - // Delete old checkpoint if exists (POC: skip actual libvirt call) - if (oldCheckpointId != null) { - // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete - logger.debug("Would delete old checkpoint: {}", oldCheckpointId); - } - - // Delete backup session record updateBackupState(backup, Backup.Status.BackedUp); backupDao.remove(backup.getId()); @@ -322,8 +332,9 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu String socket = backup.getUuid(); VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); if (vm.getState() == State.Stopped) { + Map vmDetails = vmInstanceDetailsDao.listDetailsKeyPairs(backup.getVmId()); String volumePath = getVolumePathForFileBasedBackend(volume); - startNBDServer(transferId, direction, backup.getHostId(), volume.getUuid(), volumePath, vm.getActiveCheckpointId()); + startNBDServer(transferId, direction, backup.getHostId(), volume.getUuid(), volumePath, vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID)); socket = transferId; } @@ -682,31 +693,34 @@ public List listImageTransfers(ListImageTransfersCmd cmd) return transfers.stream().map(this::toImageTransferResponse).collect(Collectors.toList()); } + private CheckpointResponse createCheckpointResponse(String checkpointId, String createTime, boolean isActive) { + CheckpointResponse response = new CheckpointResponse(); + response.setObjectName("checkpoint"); + response.setId(checkpointId); + Long createTimeSeconds = createTime != null ? NumbersUtil.parseLong(createTime, 0L) : 0L; + response.setCreated(Date.from(Instant.ofEpochSecond(createTimeSeconds))); + response.setIsActive(isActive); + return response; + } + @Override public List listVmCheckpoints(ListVmCheckpointsCmd cmd) { Long vmId = cmd.getVmId(); - VMInstanceVO vm = vmInstanceDao.findById(vmId); if (vm == null) { throw new CloudRuntimeException("VM not found: " + vmId); } - - // Return active checkpoint (POC: simplified, no libvirt query) List responses = new ArrayList<>(); - if (vm.getActiveCheckpointId() == null) { - return responses; + + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); + String activeCheckpointId = details.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID); + if (activeCheckpointId != null) { + responses.add(createCheckpointResponse(activeCheckpointId, details.get(VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME), true)); } - CheckpointResponse response = new CheckpointResponse(); - response.setObjectName("checkpoint"); - response.setId(vm.getActiveCheckpointId()); - Long createTimeSeconds = vm.getActiveCheckpointCreateTime(); - if (createTimeSeconds != null) { - response.setCreated(Date.from(Instant.ofEpochSecond(createTimeSeconds))); - } else { - response.setCreated(new Date()); + String lastCheckpointId = details.get(VmDetailConstants.LAST_CHECKPOINT_ID); + if (lastCheckpointId != null) { + responses.add(createCheckpointResponse(lastCheckpointId, details.get(VmDetailConstants.LAST_CHECKPOINT_CREATE_TIME), false)); } - response.setIsActive(true); - responses.add(response); return responses; } @@ -722,9 +736,9 @@ public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } - vm.setActiveCheckpointId(null); - vm.setActiveCheckpointCreateTime(null); - vmInstanceDao.update(cmd.getVmId(), vm); + long vmId = cmd.getVmId(); + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_ID); + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); return true; } From dc480e07d35009ddbf5c23dd36531eedfa75f02f Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:16:00 +0530 Subject: [PATCH 098/129] Implement backend for delete vm checkpoint --- .../backup/DeleteVmCheckpointCommand.java | 60 ++++++++++++++ ...bvirtDeleteVmCheckpointCommandWrapper.java | 80 +++++++++++++++++++ .../veeam/adapter/ServerAdapter.java | 1 + .../backup/KVMBackupExportServiceImpl.java | 72 ++++++++++++++++- 4 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java new file mode 100644 index 000000000000..81cf6c1abfcc --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java @@ -0,0 +1,60 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class DeleteVmCheckpointCommand extends Command { + private String vmName; + private String checkpointId; + private Map diskPathUuidMap; + private boolean stoppedVM; + + public DeleteVmCheckpointCommand() { + } + + public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, boolean stoppedVM) { + this.vmName = vmName; + this.checkpointId = checkpointId; + this.diskPathUuidMap = diskPathUuidMap; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getCheckpointId() { + return checkpointId; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java new file mode 100644 index 000000000000..edd1e09287e9 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java @@ -0,0 +1,80 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.Map; + +import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = DeleteVmCheckpointCommand.class) +public class LibvirtDeleteVmCheckpointCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return deleteBitmapsOnDisks(cmd); + } + return deleteDomainCheckpoint(cmd); + } + + private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd) { + String vmName = cmd.getVmName(); + String checkpointId = cmd.getCheckpointId(); + String virshCmd = String.format("virsh checkpoint-delete %s %s", vmName, checkpointId); + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(virshCmd); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, "Failed to delete checkpoint: " + result); + } + return new Answer(cmd, true, "Checkpoint deleted"); + } + + /** + * Stopped VM: persistent bitmaps on disk images ({@code qemu-img bitmap --remove}), matching {@link LibvirtStartBackupCommandWrapper} bitmap --add. + */ + private Answer deleteBitmapsOnDisks(DeleteVmCheckpointCommand cmd) { + String checkpointId = cmd.getCheckpointId(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + if (diskPathUuidMap == null || diskPathUuidMap.isEmpty()) { + return new Answer(cmd, false, "No disks provided for bitmap removal"); + } + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("sudo"); + script.add("qemu-img"); + script.add("bitmap"); + script.add("--remove"); + script.add(diskPath); + script.add(checkpointId); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, + "Failed to remove bitmap " + checkpointId + " from disk " + diskPath + ": " + result); + } + } + return new Answer(cmd, true, "Checkpoint bitmap removed from disks"); + } +} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 0593476b74cb..88b28b97bb76 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1712,6 +1712,7 @@ public void deleteCheckpoint(String vmUuid, String checkpointId) { DeleteVmCheckpointCmd cmd = new DeleteVmCheckpointCmd(); ComponentContext.inject(cmd); cmd.setVmId(vo.getId()); + cmd.setCheckpointId(checkpointId); kvmBackupExportService.deleteVmCheckpoint(cmd); } catch (Exception e) { throw new CloudRuntimeException("Failed to delete checkpoint: " + e.getMessage(), e); diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index e1b89b07e5ed..f7e78718bd3c 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -76,6 +76,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VmDetailConstants; @@ -203,6 +204,15 @@ public Backup startBackup(StartBackupCmd cmd) { } long hostId = backup.getHostId(); + VMInstanceDetailVO lastCheckpointId = vmInstanceDetailsDao.findDetail(vmId, VmDetailConstants.LAST_CHECKPOINT_ID); + if (lastCheckpointId != null) { + try { + sendDeleteCheckpointCommand(vm, lastCheckpointId.getValue()); + } catch (CloudRuntimeException e) { + logger.warn("Failed to delete last checkpoint {} for VM {}, proceeding with backup start", lastCheckpointId.getValue(), vmId, e); + } + } + Host host = hostDao.findById(hostId); Map vmDetails = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); String activeCkpCreateTimeStr = vmDetails.get(VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); @@ -724,9 +734,39 @@ public List listVmCheckpoints(ListVmCheckpointsCmd cmd) { return responses; } + private void sendDeleteCheckpointCommand(VMInstanceVO vm, String checkpointId) { + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + + Map diskPathUuidMap = new HashMap<>(); + if (vm.getState() == State.Stopped) { + List volumes = volumeDao.findByInstance(vm.getId()); + for (Volume vol : volumes) { + diskPathUuidMap.put(getVolumePathForFileBasedBackend(vol), vol.getUuid()); + } + } + + DeleteVmCheckpointCommand deleteCmd = new DeleteVmCheckpointCommand( + vm.getInstanceName(), + checkpointId, + diskPathUuidMap, + vm.getState() == State.Stopped); + + Answer answer; + try { + answer = agentManager.send(hostId, deleteCmd); + } catch (AgentUnavailableException | OperationTimedoutException e) { + logger.error("Failed to communicate with agent to delete checkpoint for VM {}", vm.getId(), e); + throw new CloudRuntimeException("Failed to communicate with agent", e); + } + + if (answer == null || !answer.getResult()) { + String err = answer != null ? answer.getDetails() : "null answer"; + throw new CloudRuntimeException("Failed to delete checkpoint: " + err); + } + } + @Override public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { - // Todo : backend support? VMInstanceVO vm = vmInstanceDao.findById(cmd.getVmId()); if (vm == null) { throw new CloudRuntimeException("VM not found: " + cmd.getVmId()); @@ -736,12 +776,38 @@ public boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd) { " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } + if (vm.getState() != State.Running && vm.getState() != State.Stopped) { + throw new CloudRuntimeException("VM must be running or stopped to delete checkpoint"); + } + long vmId = cmd.getVmId(); - vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_ID); - vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); + String activeCheckpointId = details.get(VmDetailConstants.ACTIVE_CHECKPOINT_ID); + if (activeCheckpointId == null || !activeCheckpointId.equals(cmd.getCheckpointId())) { + logger.error("Checkpoint ID {} to delete does not match active checkpoint ID for VM {}", cmd.getCheckpointId(), vmId); + return true; + } + + sendDeleteCheckpointCommand(vm, activeCheckpointId); + revertVmCheckpointDetailsAfterActiveDelete(vmId, details); + return true; } + private void revertVmCheckpointDetailsAfterActiveDelete(long vmId, Map detailsBeforeDelete) { + String lastId = detailsBeforeDelete.get(VmDetailConstants.LAST_CHECKPOINT_ID); + String lastTime = detailsBeforeDelete.get(VmDetailConstants.LAST_CHECKPOINT_CREATE_TIME); + if (lastId != null) { + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_ID, lastId, false); + vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME, lastTime, false); + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.LAST_CHECKPOINT_ID); + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.LAST_CHECKPOINT_CREATE_TIME); + } else { + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_ID); + vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.ACTIVE_CHECKPOINT_CREATE_TIME); + } + } + @Override public List> getCommands() { List> cmdList = new ArrayList<>(); From 6e420fecd2f98b0e7954e070c49f462ac9abeef4 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:58:36 +0530 Subject: [PATCH 099/129] fix config export to test backup apis --- .../org/apache/cloudstack/backup/KVMBackupExportService.java | 2 +- .../apache/cloudstack/backup/KVMBackupExportServiceImpl.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index fbbde961ad17..51e52c85ec34 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -52,7 +52,7 @@ public interface KVMBackupExportService extends Configurable, PluggableService { ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, "expose.kvm.backup.export.service.apis", "false", - "Enable to expose APIs for testing the KVM Backup Export Service.", false, ConfigKey.Scope.Global); + "Enable to expose APIs for testing the KVM Backup Export Service.", true, ConfigKey.Scope.Global); /** * Creates a backup session for a VM */ diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index f7e78718bd3c..c859e888ac0d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -175,6 +175,7 @@ public Backup createBackup(StartBackupCmd cmd) { backup.setToCheckpointId(toCheckpointId); backup.setFromCheckpointId(fromCheckpointId); + backup.setType("FULL"); Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); backup.setHostId(hostId); @@ -989,7 +990,8 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ ImageTransferPollingInterval, - ImageTransferIdleTimeoutSeconds + ImageTransferIdleTimeoutSeconds, + ExposeKVMBackupExportServiceApis }; } } From c588e67d6c62a0f08bbcce9a48b314ab3558d4d2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Apr 2026 12:57:52 +0530 Subject: [PATCH 100/129] changes for adding syncqueueitem for backup to block other operations on vm Signed-off-by: Abhishek Kumar --- ...ring-engine-orchestration-core-context.xml | 1 + .../backup/KVMBackupExportServiceImpl.java | 88 ++++++++++++++++++- .../backup/VmWorkWaitForBackupFinalize.java | 35 ++++++++ 3 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 17c5002c718b..49c668f50e8b 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -88,6 +88,7 @@ + diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index c859e888ac0d..7ea30035a524 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.backup; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; +import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; + import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -45,6 +48,10 @@ import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; +import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -71,23 +78,31 @@ import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.ReflectionUse; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.VmWork; +import com.cloud.vm.VmWorkConstants; +import com.cloud.vm.VmWorkJobHandler; +import com.cloud.vm.VmWorkJobHandlerProxy; +import com.cloud.vm.VmWorkSerializer; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDetailsDao; -import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; -import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; - @Component -public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService { +public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService, VmWorkJobHandler { + public static final String VM_WORK_JOB_HANDLER = KVMBackupExportServiceImpl.class.getSimpleName(); + private static final long BACKUP_FINALIZE_WAIT_CHECK_INTERVAL = 15 * 1000L; @Inject private VMInstanceDao vmInstanceDao; @@ -122,8 +137,13 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject AccountService accountService; + @Inject + AsyncJobManager asyncJobManager; + private Timer imageTransferTimer; + VmWorkJobHandlerProxy jobHandlerProxy = new VmWorkJobHandlerProxy(this); + private boolean isKVMBackupExportServiceSupported(Long zoneId) { return !BackupFrameworkEnabled.value() || StringUtils.equals("dummy", BackupProviderPlugin.valueIn(zoneId)); } @@ -189,6 +209,28 @@ protected void removeFailedBackup(BackupVO backup) { backupDao.remove(backup.getId()); } + protected void queueBackupFinalizeWaitWorkJob(final VMInstanceVO vm, final BackupVO backup) { + final CallContext context = CallContext.current(); + final Account callingAccount = context.getCallingAccount(); + final long callingUserId = context.getCallingUserId(); + + VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId()); + workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); + workJob.setCmd(VmWorkWaitForBackupFinalize.class.getName()); + workJob.setAccountId(callingAccount.getId()); + workJob.setUserId(callingUserId); + workJob.setStep(VmWorkJobVO.Step.Starting); + workJob.setVmType(VirtualMachine.Type.User); + workJob.setVmInstanceId(vm.getId()); + workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); + + VmWorkWaitForBackupFinalize workInfo = new VmWorkWaitForBackupFinalize( + callingUserId, callingAccount.getId(), vm.getId(), VM_WORK_JOB_HANDLER, backup.getId()); + workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); + + asyncJobManager.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + } + @Override public Backup startBackup(StartBackupCmd cmd) { BackupVO backup = backupDao.findById(cmd.getEntityId()); @@ -246,6 +288,7 @@ public Backup startBackup(StartBackupCmd cmd) { // Update backup with checkpoint creation time backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); updateBackupState(backup, Backup.Status.ReadyForTransfer); + queueBackupFinalizeWaitWorkJob(vm, backup); return backup; } @@ -873,6 +916,43 @@ public boolean stop() { return true; } + @ReflectionUse + public Pair orchestrateWaitForBackupFinalize(VmWorkWaitForBackupFinalize work) { + return waitForBackupTerminalState(work.getBackupId()); + } + + @Override + public Pair handleVmWorkJob(VmWork work) throws Exception { + return jobHandlerProxy.handleVmWorkJob(work); + } + + protected Pair waitForBackupTerminalState(final long backupId) { + while (true) { + final BackupVO backup = backupDao.findByIdIncludingRemoved(backupId); + if (backup == null) { + RuntimeException ex = new CloudRuntimeException(String.format("Backup %d not found while waiting for finalize", backupId)); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + + if (backup.getStatus() == Backup.Status.BackedUp) { + return new Pair<>(JobInfo.Status.SUCCEEDED, asyncJobManager.marshallResultObject(backup.getId())); + } + + if (backup.getStatus() == Backup.Status.Failed || backup.getStatus() == Backup.Status.Error) { + RuntimeException ex = new CloudRuntimeException(String.format("Backup %d reached terminal failure state: %s", backupId, backup.getStatus())); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + + try { + Thread.sleep(BACKUP_FINALIZE_WAIT_CHECK_INTERVAL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + RuntimeException ex = new CloudRuntimeException(String.format("Interrupted while waiting for backup %d finalize", backupId), e); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + } + } + private void pollImageTransferProgress() { try { List transferringTransfers = imageTransferDao.listByPhaseAndDirection( diff --git a/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java b/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java new file mode 100644 index 000000000000..ac64b47aa3ea --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.vm.VmWork; + +public class VmWorkWaitForBackupFinalize extends VmWork { + private static final long serialVersionUID = 2209426364298601717L; + + private final long backupId; + + public VmWorkWaitForBackupFinalize(long userId, long accountId, long vmId, String handlerName, long backupId) { + super(userId, accountId, vmId, handlerName); + this.backupId = backupId; + } + + public long getBackupId() { + return backupId; + } +} From 1669c0d4965a3948ccd6a251c3147a4fd58d03e1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Apr 2026 14:04:54 +0530 Subject: [PATCH 101/129] fix db list issue Signed-off-by: Abhishek Kumar --- .../com/cloud/network/dao/NetworkDaoImpl.java | 17 ++++++++++------- .../com/cloud/tags/dao/ResourceTagsDaoImpl.java | 16 ++++++++++------ .../backup/dao/ImageTransferDaoImpl.java | 16 ++++++++++------ .../cloud/api/query/dao/UserVmJoinDaoImpl.java | 16 ++++++++++------ .../cloud/api/query/dao/VolumeJoinDaoImpl.java | 16 ++++++++++------ 5 files changed, 50 insertions(+), 31 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 926e293bc2fc..a1ab1d1ef93a 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -651,19 +651,22 @@ public List listByTrafficTypeAndOwners(final TrafficType trafficType, List domainIds, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("trafficType", sb.entity().getTrafficType(), Op.EQ); - sb.and().op("account", sb.entity().getAccountId(), Op.IN); - sb.or("domain", sb.entity().getDomainId(), Op.IN); - sb.cp(); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } sb.done(); final SearchCriteria sc = sb.create(); sc.setParameters("trafficType", trafficType); - if (CollectionUtils.isNotEmpty(accountIds)) { + if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } - if (CollectionUtils.isNotEmpty(domainIds)) { - sc.setParameters("domain", domainIds); + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); } - return listBy(sc, filter); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index 47556018de4c..5dd799837665 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -128,17 +128,21 @@ public List listByResourceTypeAndOwners(ResourceObjectType resour List domainIds, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); - sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); - sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); - sb.cp(); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } sb.done(); final SearchCriteria sc = sb.create();; sc.setParameters("resourceType", resourceType); - if (CollectionUtils.isNotEmpty(accountIds)) { + if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } - if (CollectionUtils.isNotEmpty(domainIds)) { - sc.setParameters("domain", domainIds); + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); } return listBy(sc, filter); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 85dd174c129b..3e1f6b513a58 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -108,16 +108,20 @@ public List listByPhaseAndDirection(ImageTransfer.Phase phase, @Override public List listByOwners(List accountIds, List domainIds, Filter filter) { SearchBuilder sb = createSearchBuilder(); - sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); - sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); - sb.cp(); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } sb.done(); final SearchCriteria sc = sb.create(); - if (CollectionUtils.isNotEmpty(accountIds)) { + if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } - if (CollectionUtils.isNotEmpty(domainIds)) { - sc.setParameters("domain", domainIds); + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); } return listBy(sc, filter); diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 94b25ccc82f5..a0a5c1a43dda 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -835,19 +835,23 @@ public List listLeaseInstancesExpiringInDays(int days) { @Override public List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, - List accountIds, String domainPath, Filter filter) { + List accountIds, String domainPath, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("hypervisorType", sb.entity().getHypervisorType(), Op.EQ); - sb.and().op("account", sb.entity().getAccountId(), Op.IN); - sb.or("domainPath", sb.entity().getDomainPath(), Op.LIKE); - sb.cp(); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainPathNotBlank = StringUtils.isNotBlank(domainPath); + if (accountIdsNotEmpty || domainPathNotBlank) { + sb.and().op("account", sb.entity().getAccountId(), Op.IN); + sb.or("domainPath", sb.entity().getDomainPath(), Op.LIKE); + sb.cp(); + } sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("hypervisorType", hypervisorType); - if (CollectionUtils.isNotEmpty(accountIds)) { + if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } - if (StringUtils.isNotBlank(domainPath)) { + if (domainPathNotBlank) { sc.setParameters("domainPath", domainPath + "%"); } return listBy(sc, filter); diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 4bcb8bff6879..8e79dfe4b742 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -385,21 +385,25 @@ public List listByInstanceId(long instanceId) { @Override public List listByHypervisorTypeAndOwners(Hypervisor.HypervisorType hypervisorType, - List accountIds, String domainPath, Filter filter) { + List accountIds, String domainPath, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("vmType", sb.entity().getVmType(), SearchCriteria.Op.EQ); sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); - sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); - sb.or("domainPath", sb.entity().getDomainPath(), SearchCriteria.Op.LIKE); - sb.cp(); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainPathNotBlank = StringUtils.isNotBlank(domainPath); + if (accountIdsNotEmpty || domainPathNotBlank) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domainPath", sb.entity().getDomainPath(), SearchCriteria.Op.LIKE); + sb.cp(); + } sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("vmType", VirtualMachine.Type.User); sc.setParameters("hypervisorType", hypervisorType); - if (CollectionUtils.isNotEmpty(accountIds)) { + if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } - if (StringUtils.isNotBlank(domainPath)) { + if (domainPathNotBlank) { sc.setParameters("domainPath", domainPath + "%"); } return search(sc, filter); From 800faa4a6f511d8d06f7a5ab776fe2b982557c80 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Apr 2026 14:09:27 +0530 Subject: [PATCH 102/129] add log Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/backup/KVMBackupExportServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 7ea30035a524..564c791fb6be 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -942,7 +942,8 @@ protected Pair waitForBackupTerminalState(final long bac RuntimeException ex = new CloudRuntimeException(String.format("Backup %d reached terminal failure state: %s", backupId, backup.getStatus())); return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); } - + logger.debug("{} is not in a terminal state, current state: {}, waiting {}ms to check again", + backup, backup.getStatus(), BACKUP_FINALIZE_WAIT_CHECK_INTERVAL); try { Thread.sleep(BACKUP_FINALIZE_WAIT_CHECK_INTERVAL); } catch (InterruptedException e) { From e836babf0ee3a37051286710df38c7ad039b198a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Apr 2026 16:32:43 +0530 Subject: [PATCH 103/129] fix vm tags Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 29 ++++++++++++++----- .../cloudstack/veeam/api/VmsRouteHandler.java | 4 ++- .../AsyncJobJoinVOToJobConverter.java | 2 +- .../ResourceTagVOToTagConverter.java | 5 ++-- .../converter/UserVmJoinVOToVmConverter.java | 10 +++++-- .../apache/cloudstack/veeam/api/dto/Vm.java | 8 ++--- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 88b28b97bb76..7e98070c6b0f 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -357,10 +357,8 @@ protected static Tag getDummyTagByName(String name) { protected static Map getDummyTags() { Map tags = new HashMap<>(); - Tag tag1 = getDummyTagByName("Automatic"); - tags.put(tag1.getId(), tag1); - Tag tag2 = getDummyTagByName("Manual"); - tags.put(tag2.getId(), tag2); + Tag rootTag = ResourceTagVOToTagConverter.getRootTag(); + tags.put(rootTag.getId(), rootTag); return tags; } @@ -696,7 +694,7 @@ protected Vm createInstance(com.cloud.dc.DataCenter zone, Long clusterId, Accoun vm = userVmManager.finalizeCreateVirtualMachine(vm.getId()); UserVmJoinVO vo = userVmJoinDao.findById(vm.getId()); return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, - this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); + this::listTagsByInstanceId, this::listDiskAttachmentsByInstanceId, this::listNicsByInstance, false); } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException | CloudRuntimeException e) { throw new CloudRuntimeException("Failed to create VM: " + e.getMessage(), e); } @@ -983,13 +981,15 @@ public List listAllInstances(Long offset, Long limit) { } @ApiAccess(command = ListVMsCmd.class) - public Vm getInstance(String uuid, boolean includeDisks, boolean includeNics, boolean allContent) { + public Vm getInstance(String uuid, boolean includeTags, boolean includeDisks, boolean includeNics, + boolean allContent) { UserVmJoinVO vo = userVmJoinDao.findByUuid(uuid); if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, this::getDetailsByInstanceId, + includeTags ? this::listTagsByInstanceId : null, includeDisks ? this::listDiskAttachmentsByInstanceId : null, includeNics ? this::listNicsByInstance : null, allContent); @@ -1064,7 +1064,7 @@ public Vm createInstance(Vm request) { @ApiAccess(command = UpdateVMCmd.class) public Vm updateInstance(String uuid, Vm request) { logger.warn("Received request to update VM with ID {}. No action, returning existing VM data.", uuid); - return getInstance(uuid, false, false, false); + return getInstance(uuid, false, false, false, false); } @ApiAccess(command = DestroyVMCmd.class) @@ -1201,6 +1201,21 @@ public Disk reduceDisk(String uuid) { throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); } + @ApiAccess(command = ListTagsCmd.class) + protected List listTagsByInstanceId(final long instanceId) { + List vmResourceTags = resourceTagDao.listBy(instanceId, + ResourceTag.ResourceObjectType.UserVm); + List tags = new ArrayList<>(); + for (ResourceTag t : vmResourceTags) { + if (t instanceof ResourceTagVO) { + tags.add((ResourceTagVO)t); + continue; + } + tags.add(resourceTagDao.findById(t.getId())); + } + return ResourceTagVOToTagConverter.toTags(tags); + } + @ApiAccess(command = ListVolumesCmd.class) protected List listDiskAttachmentsByInstanceId(final long instanceId) { List kvmVolumes = volumeJoinDao.listByInstanceId(instanceId); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index a3d1ca236cbf..0b5aff478234 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -256,6 +256,7 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons protected void handleGetById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { String followStr = req.getParameter("follow"); + boolean includeTags = false; boolean includeDisks = false; boolean includeNics = false; if (StringUtils.isNotBlank(followStr)) { @@ -263,12 +264,13 @@ protected void handleGetById(final String id, final HttpServletRequest req, fina .map(String::trim) .filter(s -> !s.isEmpty()) .collect(java.util.stream.Collectors.toSet()); + includeTags = followParts.contains("tags"); includeDisks = followParts.contains("disk_attachments.disk"); includeNics = followParts.contains("nics.reporteddevices"); } boolean allContent = Boolean.parseBoolean(req.getParameter("all_content")); try { - Vm response = serverAdapter.getInstance(id, includeDisks, includeNics, allContent); + Vm response = serverAdapter.getInstance(id, includeTags, includeDisks, includeNics, allContent); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index dc2853dfd766..f8845804e8ef 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -98,7 +98,7 @@ protected static void fillAction(final ResourceAction action, final AsyncJobJoin public static VmAction toVmAction(final AsyncJobJoinVO vo, final UserVmJoinVO vm) { VmAction action = new VmAction(); fillAction(action, vo); - action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null, null, false)); + action.setVm(UserVmJoinVOToVmConverter.toVm(vm, null, null, null, null, null, false)); return action; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java index d22a234d9e47..445b3c0ae33a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java @@ -38,7 +38,6 @@ public static Ref getRootTagRef() { } public static Tag getRootTag() { - String basePath = VeeamControlService.ContextPath.value(); Tag tag = new Tag(); tag.setId(BaseDto.ZERO_UUID); tag.setName("root"); @@ -50,8 +49,8 @@ public static Tag toTag(ResourceTagVO vo) { String basePath = VeeamControlService.ContextPath.value(); Tag tag = new Tag(); tag.setId(vo.getUuid()); - tag.setName(vo.getKey()); - tag.setDescription(String.format("Tag %s-%s", vo.getKey(), vo.getValue())); + tag.setName(String.format("%s-%s", vo.getKey(), vo.getValue()).replaceAll("\\s+", "")); + tag.setDescription(String.format("Tag %s with value: %s", vo.getKey(), vo.getValue())); tag.setHref(basePath + TagsRouteHandler.BASE_ROUTE + "/" + vo.getUuid()); if (ResourceTag.ResourceObjectType.UserVm.equals(vo.getResourceType())) { tag.setVm(Ref.of(basePath + VmsRouteHandler.BASE_ROUTE + "/" + vo.getResourceUuid(), diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 7f148b8d65b9..61269ab04109 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -31,12 +31,12 @@ import org.apache.cloudstack.veeam.api.dto.BaseDto; import org.apache.cloudstack.veeam.api.dto.Cpu; import org.apache.cloudstack.veeam.api.dto.DiskAttachment; -import org.apache.cloudstack.veeam.api.dto.EmptyElement; import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Nic; import org.apache.cloudstack.veeam.api.dto.Os; import org.apache.cloudstack.veeam.api.dto.OvfXmlUtil; import org.apache.cloudstack.veeam.api.dto.Ref; +import org.apache.cloudstack.veeam.api.dto.Tag; import org.apache.cloudstack.veeam.api.dto.Topology; import org.apache.cloudstack.veeam.api.dto.Vm; import org.apache.commons.collections.MapUtils; @@ -58,6 +58,7 @@ private UserVmJoinVOToVmConverter() { */ public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, final Function> detailsResolver, + final Function> tagsResolver, final Function> disksResolver, final Function> nicsResolver, final boolean allContent) { @@ -160,7 +161,10 @@ public static Vm toVm(final UserVmJoinVO src, final Function h BaseDto.getActionLink("reporteddevices", dst.getHref()), BaseDto.getActionLink("snapshots", dst.getHref()) )); - dst.setTags(new EmptyElement()); + if (tagsResolver != null) { + List tags = tagsResolver.apply(src.getId()); + dst.setTags(NamedList.of("tag", tags)); + } dst.setCpuProfile(Ref.of( basePath + ApiService.BASE_ROUTE + "/cpuprofiles/" + src.getServiceOfferingUuid(), src.getServiceOfferingUuid())); @@ -189,7 +193,7 @@ private static Vm.Initialization getOvfInitialization(Vm vm, UserVmJoinVO vo) { public static List toVmList(final List srcList, final Function hostResolver, final Function> detailsResolver) { return srcList.stream() - .map(v -> toVm(v, hostResolver, detailsResolver, null, null, false)) + .map(v -> toVm(v, hostResolver, detailsResolver, null, null, null, false)) .collect(Collectors.toList()); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index b939224d8740..90a50207aacc 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -58,8 +58,8 @@ public final class Vm extends BaseDto { private String origin; // "ovirt" private NamedList actions; // actions.link[] @JacksonXmlElementWrapper(useWrapping = false) - private List link; // related resources - private EmptyElement tags; // empty + private List link; + private NamedList tags; private NamedList diskAttachments; private NamedList nics; private Initialization initialization; @@ -252,11 +252,11 @@ public void setLink(List link) { this.link = link; } - public EmptyElement getTags() { + public NamedList getTags() { return tags; } - public void setTags(EmptyElement tags) { + public void setTags(NamedList tags) { this.tags = tags; } From 10782021e4fb0c0bdd5ad334d99eb1fc06b8c9b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Apr 2026 17:39:02 +0530 Subject: [PATCH 104/129] addressed with finalizing transfers before backup Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 564c791fb6be..036ecd6c16e5 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -872,7 +872,6 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans response.setId(imageTransferVO.getUuid()); Long backupId = imageTransferVO.getBackupId(); if (backupId != null) { - // ToDo: Orphan image transfer record if backup is deleted before transfer finalization, need to clean up Backup backup = backupDao.findByIdIncludingRemoved(backupId); response.setBackupId(backup.getUuid()); } From 1ddccaa767da2fe86bcb1af4ce9f3de1fca12b75 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Apr 2026 09:48:09 +0530 Subject: [PATCH 105/129] fix storagedomain retrieval Signed-off-by: Abhishek Kumar --- .../java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 5f4527a7c555..fe040f8011e0 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -416,7 +416,7 @@ public List findStoragePoolByScopeAndRuleTags(Long datacenterId, public List listByZoneAndType(long zoneId, List types, Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("types", sb.entity().getZoneId(), SearchCriteria.Op.IN); + sb.and("types", sb.entity().getPoolType(), SearchCriteria.Op.IN); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("zoneId", zoneId); From f118fc2f81a55133b3ed94bca35b7117f90e08fb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Apr 2026 15:13:48 +0530 Subject: [PATCH 106/129] add logs for unimplemented endpoints Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/veeam/api/DisksRouteHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 0dd316753552..581937b56ef0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -165,11 +165,13 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, protected void handlePutById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req, logger); throw new InvalidParameterValueException("Put Disk with ID " + id + " not implemented"); } protected void handlePostDiskCopy(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req, logger); try { Disk response = serverAdapter.copyDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); @@ -180,6 +182,7 @@ protected void handlePostDiskCopy(final String id, final HttpServletRequest req, protected void handlePostDiskReduce(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { + String data = RouteHandler.getRequestData(req, logger); try { Disk response = serverAdapter.reduceDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); From b7f8fa365d5590144351644d761c78f26354cab0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Apr 2026 16:14:54 +0530 Subject: [PATCH 107/129] handle PUT on disks/{id}; refactor Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 85 +++++++++++-------- .../veeam/api/DisksRouteHandler.java | 16 +++- .../veeam/api/ImageTransfersRouteHandler.java | 2 + .../veeam/api/JobsRouteHandler.java | 2 + .../veeam/api/NetworksRouteHandler.java | 2 + .../veeam/api/TagsRouteHandler.java | 2 + .../cloudstack/veeam/api/VmsRouteHandler.java | 30 +++++-- .../veeam/api/VnicProfilesRouteHandler.java | 2 + 8 files changed, 95 insertions(+), 46 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 7e98070c6b0f..36252f583838 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -82,6 +82,7 @@ import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.UpdateVolumeCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -1174,33 +1175,6 @@ public VmAction shutdownInstance(String uuid, boolean async) { } } - @ApiAccess(command = ListVolumesCmd.class) - public List listAllDisks(Long offset, Long limit) { - Filter filter = new Filter(VolumeJoinVO.class, "id", true, offset, limit); - Pair, String> ownerDetails = getResourceOwnerFilters(); - List kvmVolumes = volumeJoinDao.listByHypervisorTypeAndOwners(Hypervisor.HypervisorType.KVM, - ownerDetails.first(), ownerDetails.second(), filter); - return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes, this::getVolumePhysicalSize); - } - - @ApiAccess(command = ListVolumesCmd.class) - public Disk getDisk(String uuid) { - VolumeVO vo = volumeDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); - } - accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); - return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findByUuid(uuid), this::getVolumePhysicalSize); - } - - public Disk copyDisk(String uuid) { - throw new InvalidParameterValueException("Copy Disk with ID " + uuid + " not implemented"); - } - - public Disk reduceDisk(String uuid) { - throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); - } - @ApiAccess(command = ListTagsCmd.class) protected List listTagsByInstanceId(final long instanceId) { List vmResourceTags = resourceTagDao.listBy(instanceId, @@ -1232,6 +1206,25 @@ public List listDiskAttachmentsByInstanceUuid(final String uuid) return listDiskAttachmentsByInstanceId(vo.getId()); } + @ApiAccess(command = ListVolumesCmd.class) + public List listAllDisks(Long offset, Long limit) { + Filter filter = new Filter(VolumeJoinVO.class, "id", true, offset, limit); + Pair, String> ownerDetails = getResourceOwnerFilters(); + List kvmVolumes = volumeJoinDao.listByHypervisorTypeAndOwners(Hypervisor.HypervisorType.KVM, + ownerDetails.first(), ownerDetails.second(), filter); + return VolumeJoinVOToDiskConverter.toDiskList(kvmVolumes, this::getVolumePhysicalSize); + } + + @ApiAccess(command = ListVolumesCmd.class) + public Disk getDisk(String uuid) { + VolumeVO vo = volumeDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, vo); + return VolumeJoinVOToDiskConverter.toDisk(volumeJoinDao.findByUuid(uuid), this::getVolumePhysicalSize); + } + protected void assignVolumeToAccount(VolumeVO volumeVO, long accountId) { Account account = accountService.getActiveAccountById(accountId); if (account == null) { @@ -1296,15 +1289,6 @@ public DiskAttachment attachInstanceDisk(final String vmUuid, final DiskAttachme return VolumeJoinVOToDiskConverter.toDiskAttachment(attachedVolumeVO, this::getVolumePhysicalSize); } - @ApiAccess(command = DestroyVolumeCmd.class) - public void deleteDisk(String uuid) { - VolumeVO vo = volumeDao.findByUuid(uuid); - if (vo == null) { - throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); - } - volumeApiService.deleteVolume(vo.getId(), accountService.getSystemAccount()); - } - @ApiAccess(command = CreateVolumeCmd.class) public Disk createDisk(Disk request) { if (request == null) { @@ -1346,6 +1330,35 @@ public Disk createDisk(Disk request) { return createDisk(caller, pool, name, diskOfferingId, provisionedSizeInGb, initialSize); } + @ApiAccess(command = DestroyVolumeCmd.class) + public void deleteDisk(String uuid) { + VolumeVO vo = volumeDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + volumeApiService.deleteVolume(vo.getId(), accountService.getSystemAccount()); + } + + @ApiAccess(command = UpdateVolumeCmd.class) + public Disk updateDisk(String uuid, Disk request) { + VolumeVO vo = volumeDao.findByUuid(uuid); + if (vo == null) { + throw new InvalidParameterValueException("Disk with ID " + uuid + " not found"); + } + logger.warn("Update disk is not implemented, returning disk ID: {} as it is", uuid); + return getDisk(uuid); + } + + @ApiAccess(command = UpdateVolumeCmd.class) + public Disk copyDisk(String uuid) { + throw new InvalidParameterValueException("Copy Disk with ID " + uuid + " not implemented"); + } + + @ApiAccess(command = UpdateVolumeCmd.class) + public Disk reduceDisk(String uuid) { + throw new InvalidParameterValueException("Reduce Disk with ID " + uuid + " not implemented"); + } + @ApiAccess(command = ListNicsCmd.class) public List listNicsByInstanceUuid(final String uuid) { UserVmVO vo = userVmDao.findByUuid(uuid); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java index 581937b56ef0..d12745769e1b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java @@ -150,6 +150,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -158,7 +160,7 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, try { serverAdapter.deleteDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, "Deleted disk ID: " + id, outFormat); - } catch (InvalidParameterValueException e) { + } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } } @@ -166,7 +168,13 @@ protected void handleDeleteById(final String id, final HttpServletResponse resp, protected void handlePutById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { String data = RouteHandler.getRequestData(req, logger); - throw new InvalidParameterValueException("Put Disk with ID " + id + " not implemented"); + try { + Disk request = io.getMapper().jsonMapper().readValue(data, Disk.class); + Disk response = serverAdapter.updateDisk(id, request); + io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); + } catch (JsonProcessingException | CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); + } } protected void handlePostDiskCopy(final String id, final HttpServletRequest req, final HttpServletResponse resp, @@ -175,7 +183,7 @@ protected void handlePostDiskCopy(final String id, final HttpServletRequest req, try { Disk response = serverAdapter.copyDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); - } catch (InvalidParameterValueException e) { + } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } } @@ -186,7 +194,7 @@ protected void handlePostDiskReduce(final String id, final HttpServletRequest re try { Disk response = serverAdapter.reduceDisk(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); - } catch (InvalidParameterValueException e) { + } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java index ef27d6353fbf..1a04e4028cf0 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java @@ -134,6 +134,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi ImageTransfer response = serverAdapter.getImageTransfer(id); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { + io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { io.badRequest(resp, e.getMessage(), outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java index 50a7465414e6..95e4e3c9559a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java @@ -101,6 +101,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java index 31e5bccca7c7..2d1f0962c2b6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java @@ -103,6 +103,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java index 727cf72ca1a5..e1daefc1c443 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java @@ -104,6 +104,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 0b5aff478234..a2d720c4864d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -274,6 +274,8 @@ protected void handleGetById(final String id, final HttpServletRequest req, fina io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -284,8 +286,8 @@ protected void handleUpdateById(final String id, final HttpServletRequest req, f Vm request = io.getMapper().jsonMapper().readValue(data, Vm.class); Vm response = serverAdapter.updateInstance(id, request); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); - } catch (InvalidParameterValueException e) { - io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -296,7 +298,7 @@ protected void handleDeleteById(final String id, final HttpServletRequest req, f VmAction vm = serverAdapter.deleteInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_OK, vm, outFormat); } catch (CloudRuntimeException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -307,7 +309,7 @@ protected void handleStartVmById(final String id, final HttpServletRequest req, VmAction vm = serverAdapter.startInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -318,7 +320,7 @@ protected void handleStopVmById(final String id, final HttpServletRequest req, f VmAction vm = serverAdapter.stopInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -329,7 +331,7 @@ protected void handleShutdownVmById(final String id, final HttpServletRequest re VmAction vm = serverAdapter.shutdownInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); } catch (CloudRuntimeException e) { - io.notFound(resp, e.getMessage(), outFormat); + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -341,6 +343,8 @@ protected void handleGetDiskAttachmentsByVmId(final String id, final HttpServlet io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -365,6 +369,8 @@ protected void handleGetNicsByVmId(final String id, final HttpServletResponse re io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -389,6 +395,8 @@ protected void handleGetSnapshotsByVmId(final String id, final HttpServletRespon io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -412,6 +420,8 @@ protected void handleGetSnapshotById(final String id, final HttpServletResponse io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -452,6 +462,8 @@ protected void handleGetBackupsByVmId(final String id, final HttpServletResponse io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -475,6 +487,8 @@ protected void handleGetBackupById(final String id, final HttpServletResponse re io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -487,6 +501,8 @@ protected void handleGetBackupDisksById(final String id, final HttpServletReques io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } @@ -509,6 +525,8 @@ protected void handleGetCheckpointsByVmId(final String id, final HttpServletResp io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java index 31fb93ddf3b7..fbfc0c9a92dd 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java @@ -103,6 +103,8 @@ protected void handleGetById(final String id, final HttpServletResponse resp, fi io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); + } catch (CloudRuntimeException e) { + io.badRequest(resp, e.getMessage(), outFormat); } } } From 259ba31e90237a701d51c98546b8e99a6111ea6e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Apr 2026 11:12:17 +0530 Subject: [PATCH 108/129] fix vms listing with tags, effectively tagged jobs Signed-off-by: Abhishek Kumar --- .../veeam/adapter/ServerAdapter.java | 14 +++++++-- .../veeam/api/DataCentersRouteHandler.java | 4 +-- .../cloudstack/veeam/api/VmsRouteHandler.java | 30 ++++++++----------- .../converter/UserVmJoinVOToVmConverter.java | 24 +++++++++------ .../veeam/api/request/ListQuery.java | 29 ++++++++++++++---- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 36252f583838..706752c62819 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -973,12 +973,19 @@ public VnicProfile getVnicProfile(String uuid) { } @ApiAccess(command = ListVMsCmd.class) - public List listAllInstances(Long offset, Long limit) { + public List listAllInstances(boolean includeTags, boolean includeDisks, boolean includeNics, + boolean allContent, Long offset, Long limit) { Filter filter = new Filter(UserVmJoinVO.class, "id", true, offset, limit); Pair, String> ownerDetails = getResourceOwnerFilters(); List vms = userVmJoinDao.listByHypervisorTypeAndOwners(Hypervisor.HypervisorType.KVM, ownerDetails.first(), ownerDetails.second(), filter); - return UserVmJoinVOToVmConverter.toVmList(vms, this::getHostById, this::getDetailsByInstanceId); + return UserVmJoinVOToVmConverter.toVmList(vms, + this::getHostById, + this::getDetailsByInstanceId, + includeTags ? this::listTagsByInstanceId : null, + includeDisks ? this::listDiskAttachmentsByInstanceId : null, + includeNics ? this::listNicsByInstance : null, + allContent); } @ApiAccess(command = ListVMsCmd.class) @@ -988,7 +995,8 @@ public Vm getInstance(String uuid, boolean includeTags, boolean includeDisks, bo if (vo == null) { throw new InvalidParameterValueException("VM with ID " + uuid + " not found"); } - return UserVmJoinVOToVmConverter.toVm(vo, this::getHostById, + return UserVmJoinVOToVmConverter.toVm(vo, + this::getHostById, this::getDetailsByInstanceId, includeTags ? this::listTagsByInstanceId : null, includeDisks ? this::listDiskAttachmentsByInstanceId : null, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java index 7e68375fe567..a06af4f24429 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java @@ -122,7 +122,7 @@ protected void handleGetStorageDomainsByDcId(final String id, final HttpServletR throws IOException { try { ListQuery query = ListQuery.fromRequest(req); - List storageDomains = serverAdapter.listStorageDomainsByDcId(id, query.getPage(), + List storageDomains = serverAdapter.listStorageDomainsByDcId(id, query.getOffset(), query.getMax()); NamedList response = NamedList.of("storage_domain", storageDomains); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); @@ -138,7 +138,7 @@ protected void handleGetNetworksByDcId(final String id, final HttpServletRequest throws IOException { try { ListQuery query = ListQuery.fromRequest(req); - List networks = serverAdapter.listNetworksByDcId(id, query.getPage(), + List networks = serverAdapter.listNetworksByDcId(id, query.getOffset(), query.getMax()); NamedList response = NamedList.of("network", networks); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index a2d720c4864d..92156be5e699 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.List; -import java.util.Set; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -42,7 +41,6 @@ import org.apache.cloudstack.veeam.utils.Negotiation; import org.apache.cloudstack.veeam.utils.PathUtil; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; @@ -233,7 +231,12 @@ protected void handleGet(final HttpServletRequest req, final HttpServletResponse Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { try { ListQuery query = ListQuery.fromRequest(req); - final List result = serverAdapter.listAllInstances(query.getOffset(), query.getLimit()); + final List result = serverAdapter.listAllInstances(query.followContains("tags"), + query.followContains("disk_attachments.disk"), + query.followContains("nics.reporteddevices"), + query.isAllContent(), + query.getOffset(), + query.getLimit()); NamedList response = NamedList.of("vm", result); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (CloudRuntimeException e) { @@ -255,22 +258,13 @@ protected void handlePost(final HttpServletRequest req, final HttpServletRespons protected void handleGetById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - String followStr = req.getParameter("follow"); - boolean includeTags = false; - boolean includeDisks = false; - boolean includeNics = false; - if (StringUtils.isNotBlank(followStr)) { - Set followParts = java.util.Arrays.stream(followStr.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(java.util.stream.Collectors.toSet()); - includeTags = followParts.contains("tags"); - includeDisks = followParts.contains("disk_attachments.disk"); - includeNics = followParts.contains("nics.reporteddevices"); - } - boolean allContent = Boolean.parseBoolean(req.getParameter("all_content")); try { - Vm response = serverAdapter.getInstance(id, includeTags, includeDisks, includeNics, allContent); + ListQuery query = ListQuery.fromRequest(req); + Vm response = serverAdapter.getInstance(id, + query.followContains("tags"), + query.followContains("disk_attachments.disk"), + query.followContains("nics.reporteddevices"), + query.isAllContent()); io.getWriter().write(resp, HttpServletResponse.SC_OK, response, outFormat); } catch (InvalidParameterValueException e) { io.notFound(resp, e.getMessage(), outFormat); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java index 61269ab04109..dafec627e963 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java @@ -56,12 +56,13 @@ private UserVmJoinVOToVmConverter() { * * @param src UserVmJoinVO */ - public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, - final Function> detailsResolver, - final Function> tagsResolver, - final Function> disksResolver, - final Function> nicsResolver, - final boolean allContent) { + public static Vm toVm(final UserVmJoinVO src, + final Function hostResolver, + final Function> detailsResolver, + final Function> tagsResolver, + final Function> disksResolver, + final Function> nicsResolver, + final boolean allContent) { if (src == null) { return null; } @@ -190,10 +191,15 @@ private static Vm.Initialization getOvfInitialization(Vm vm, UserVmJoinVO vo) { return initialization; } - public static List toVmList(final List srcList, final Function hostResolver, - final Function> detailsResolver) { + public static List toVmList(final List srcList, + final Function hostResolver, + final Function> detailsResolver, + final Function> tagsResolver, + final Function> disksResolver, + final Function> nicsResolver, + final boolean allContent) { return srcList.stream() - .map(v -> toVm(v, hostResolver, detailsResolver, null, null, null, false)) + .map(v -> toVm(v, hostResolver, detailsResolver, tagsResolver, disksResolver, nicsResolver, allContent)) .collect(Collectors.toList()); } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java index 8a21b595b770..f57edf76e04c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java @@ -17,11 +17,15 @@ package org.apache.cloudstack.veeam.api.request; +import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; @@ -31,6 +35,7 @@ public class ListQuery { Long max; Long page; Map search; + List follow; public boolean isAllContent() { return allContent; @@ -48,16 +53,19 @@ public void setMax(Long max) { this.max = max; } - public Map getSearch() { - return search; - } - public void setSearch(Map search) { this.search = search; } - public Long getPage() { - return page; + public void setFollow(String followStr) { + if (StringUtils.isBlank(followStr)) { + this.follow = null; + return; + } + this.follow = Arrays.stream(followStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); } public Long getOffset() { @@ -71,6 +79,13 @@ public Long getLimit() { return max; } + public boolean followContains(String part) { + if (CollectionUtils.isEmpty(follow)) { + return false; + } + return follow.contains(part); + } + public static ListQuery fromRequest(HttpServletRequest request) { ListQuery query = new ListQuery(); if (MapUtils.isEmpty(request.getParameterMap())) { @@ -89,6 +104,8 @@ public static ListQuery fromRequest(HttpServletRequest request) { // Ignore invalid max and keep default null value. } } + String follow = request.getParameter("follow"); + query.setFollow(follow); Map searchItems = getSearchMap(request.getParameter("search")); if (!searchItems.isEmpty()) { try { From d804b7597bc05c251ef273cee3ddef2560207ecb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Apr 2026 12:22:35 +0530 Subject: [PATCH 109/129] address orphan trnasfers Signed-off-by: Abhishek Kumar --- .../backup/KVMBackupExportServiceImpl.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 036ecd6c16e5..3b160ce4885a 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -637,11 +637,7 @@ public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTran @Override public boolean cancelImageTransfer(long imageTransferId) { - ImageTransferVO imageTransfer = imageTransferDao.findById(imageTransferId); - if (imageTransfer == null) { - throw new CloudRuntimeException("Image transfer not found: " + imageTransferId); - } - // ToDo: Implement cancel logic + finalizeImageTransfer(imageTransferId); return true; } @@ -876,7 +872,6 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans response.setBackupId(backup.getUuid()); } Long volumeId = imageTransferVO.getDiskId(); - // ToDo: fix volume deletion leaving orphan image transfer record Volume volume = volumeDao.findByIdIncludingRemoved(volumeId); response.setDiskId(volume.getUuid()); response.setTransferUrl(imageTransferVO.getTransferUrl()); @@ -977,7 +972,8 @@ private void pollImageTransferProgress() { for (ImageTransferVO transfer : hostTransfers) { VolumeVO volume = volumeDao.findById(transfer.getDiskId()); if (volume == null) { - logger.warn("Volume not found for image transfer: " + transfer.getUuid()); + logger.warn("Volume not found for image transfer: {}", transfer.getUuid()); + imageTransferDao.remove(transfer.getId()); // ToDo: confirm if this enough? continue; } transferVolumeMap.put(transfer.getId(), volume); @@ -986,7 +982,7 @@ private void pollImageTransferProgress() { transferIds.add(transferId); if (volume.getPath() == null) { - logger.warn("Volume path is null for image transfer: " + transfer.getUuid()); + logger.warn("Volume path is null for image transfer: {}", transfer.getUuid()); continue; } String volumePath = getVolumePathForFileBasedBackend(volume); @@ -1004,7 +1000,7 @@ private void pollImageTransferProgress() { if (answer == null || !answer.getResult() || MapUtils.isEmpty(answer.getProgressMap())) { logger.warn("Failed to get progress for transfers on host {}: {}", hostId, answer != null ? answer.getDetails() : "null answer"); - return; + return; // ToDo: return on continue? } for (ImageTransferVO transfer : hostTransfers) { String transferId = transfer.getUuid(); From 2f673568aa92fa8673932f95b68674601c53a3a3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Apr 2026 18:25:14 +0530 Subject: [PATCH 110/129] cleanup; return tags with specific key only Signed-off-by: Abhishek Kumar --- .../com/cloud/tags/dao/ResourceTagDao.java | 6 +- .../cloud/tags/dao/ResourceTagsDaoImpl.java | 24 +++++++- .../META-INF/db/views/cloud.user_vm_view.sql | 1 + .../apache/cloudstack/veeam/RouteHandler.java | 8 ++- .../cloudstack/veeam/VeeamControlServer.java | 6 +- .../cloudstack/veeam/VeeamControlService.java | 15 +++++ .../cloudstack/veeam/VeeamControlServlet.java | 17 +----- .../veeam/adapter/ServerAdapter.java | 56 +++++++++---------- .../{ApiService.java => ApiRouteHandler.java} | 38 ++++--------- .../cloudstack/veeam/api/VmsRouteHandler.java | 17 ++---- .../AsyncJobJoinVOToJobConverter.java | 20 ------- .../converter/BackupVOToBackupConverter.java | 13 +++++ .../ClusterVOToClusterConverter.java | 19 ++----- ...DataCenterJoinVOToDataCenterConverter.java | 10 ++-- .../converter/HostJoinVOToHostConverter.java | 23 +++----- .../NetworkVOToNetworkConverter.java | 1 - .../NetworkVOToVnicProfileConverter.java | 1 - .../api/converter/NicVOToNicConverter.java | 6 +- .../ResourceTagVOToTagConverter.java | 7 ++- .../StoreVOToStorageDomainConverter.java | 6 +- .../converter/UserVmJoinVOToVmConverter.java | 25 ++++----- .../VolumeJoinVOToDiskConverter.java | 4 +- .../cloudstack/veeam/api/dto/Backup.java | 18 ++++++ .../cloudstack/veeam/api/dto/BaseDto.java | 6 ++ .../apache/cloudstack/veeam/api/dto/Host.java | 4 ++ .../cloudstack/veeam/api/dto/Version.java | 21 +++++++ .../apache/cloudstack/veeam/api/dto/Vm.java | 5 +- .../veeam/api/response/FaultResponse.java | 39 ------------- .../cloudstack/veeam/utils/PathUtil.java | 3 +- .../veeam/utils/ResponseWriter.java | 3 +- .../spring-veeam-control-service-context.xml | 2 +- .../main/resources/{test.xml => test-ovf.xml} | 18 ++++++ .../com/cloud/api/query/vo/UserVmJoinVO.java | 7 +++ 33 files changed, 229 insertions(+), 220 deletions(-) rename plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/{ApiService.java => ApiRouteHandler.java} (80%) delete mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java rename plugins/integrations/veeam-control-service/src/main/resources/{test.xml => test-ovf.xml} (92%) diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index 3b946eba9622..034ea61ee0e9 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -63,6 +63,8 @@ public interface ResourceTagDao extends GenericDao { List listByResourceUuid(String resourceUuid); - List listByResourceTypeAndOwners(ResourceObjectType resourceType, List accountIds, - List domainIds, Filter filter); + List listByResourceTypeKeyAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, Filter filter); + + ResourceTagVO findByResourceTypeKeyAndValue(ResourceObjectType resourceType, String key, String value); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index 5dd799837665..b82dd5ec3dec 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -124,10 +124,12 @@ public List listByResourceUuid(String resourceUuid) { } @Override - public List listByResourceTypeAndOwners(ResourceObjectType resourceType, List accountIds, - List domainIds, Filter filter) { + public List listByResourceTypeKeyAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter) { SearchBuilder sb = createSearchBuilder(); sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.EQ); boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); if (accountIdsNotEmpty || domainIdsNotEmpty) { @@ -136,8 +138,9 @@ public List listByResourceTypeAndOwners(ResourceObjectType resour sb.cp(); } sb.done(); - final SearchCriteria sc = sb.create();; + final SearchCriteria sc = sb.create(); sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key); if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } @@ -146,4 +149,19 @@ public List listByResourceTypeAndOwners(ResourceObjectType resour } return listBy(sc, filter); } + + @Override + public ResourceTagVO findByResourceTypeKeyAndValue(ResourceObjectType resourceType, String key, + String value) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.EQ); + sb.and("value", sb.entity().getValue(), Op.EQ); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key); + sc.setParameters("value", value); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 6f31fc17bce7..db3fd8be4841 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -56,6 +56,7 @@ SELECT `vm_instance`.`display_vm` AS `display_vm`, `vm_instance`.`delete_protection` AS `delete_protection`, `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_display_name`, `vm_instance`.`pod_id` AS `pod_id`, `host_pod_ref`.`uuid` AS `pod_uuid`, `vm_instance`.`private_ip_address` AS `private_ip_address`, diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java index d59ef9e2f795..693bfb287c68 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java @@ -19,7 +19,6 @@ import java.io.BufferedReader; import java.io.IOException; -import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -30,7 +29,6 @@ import com.cloud.utils.component.Adapter; public interface RouteHandler extends Adapter { - static final Pattern PAGE_PATTERN = Pattern.compile("\\bpage\\s+(\\d+)"); default int priority() { return 0; } boolean canHandle(String method, String path) throws IOException; void handle(HttpServletRequest req, HttpServletResponse resp, String path, Negotiation.OutFormat outFormat, VeeamControlServlet io) @@ -60,7 +58,6 @@ static String getRequestData(HttpServletRequest req) { if (!"application/json".equals(mime) && !"application/x-www-form-urlencoded".equals(mime)) { return null; } - String result = null; try { StringBuilder data = new StringBuilder(); String line; @@ -74,4 +71,9 @@ static String getRequestData(HttpServletRequest req) { return null; } } + + static boolean isRequestAsync(HttpServletRequest req) { + String asyncStr = req.getParameter("async"); + return Boolean.TRUE.toString().equals(asyncStr); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java index 3121fd6ecf4c..a70babe9b279 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -28,7 +28,7 @@ import javax.servlet.http.HttpServletRequest; import org.apache.cloudstack.utils.server.ServerPropertiesUtil; -import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.ApiRouteHandler; import org.apache.cloudstack.veeam.filter.AllowedClientCidrsFilter; import org.apache.cloudstack.veeam.filter.BearerOrBasicAuthFilter; import org.apache.commons.lang3.StringUtils; @@ -125,12 +125,12 @@ public void startIfEnabled() throws Exception { // CIDR filter for all routes AllowedClientCidrsFilter cidrFilter = new AllowedClientCidrsFilter(veeamControlService); FilterHolder cidrHolder = new FilterHolder(cidrFilter); - ctx.addFilter(cidrHolder, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); + ctx.addFilter(cidrHolder, ApiRouteHandler.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); // Bearer or Basic Auth for all routes BearerOrBasicAuthFilter authFilter = new BearerOrBasicAuthFilter(veeamControlService); FilterHolder authHolder = new FilterHolder(authFilter); - ctx.addFilter(authHolder, ApiService.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); + ctx.addFilter(authHolder, ApiRouteHandler.BASE_ROUTE + "/*", EnumSet.of(DispatcherType.REQUEST)); // Front controller servlet ctx.addServlet(new ServletHolder(new VeeamControlServlet(routeHandlers)), "/*"); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java index 8e4abef9743f..159d7eead066 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -21,10 +21,13 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.utils.CloudStackVersion; import com.cloud.utils.component.PluggableService; public interface VeeamControlService extends PluggableService, Configurable { + String PLUGIN_NAME = "CloudStack Veeam Control Service"; + ConfigKey Enabled = new ConfigKey<>("Advanced", Boolean.class, "integration.veeam.control.enabled", "false", "Enable the Veeam Integration REST API server", false); ConfigKey BindAddress = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.bind.address", @@ -57,4 +60,16 @@ public interface VeeamControlService extends PluggableService, Configurable { List getAllowedClientCidrs(); boolean validateCredentials(String username, String password); + + static String getPackageVersion() { + return VeeamControlService.class.getPackage().getImplementationVersion(); + } + + static CloudStackVersion getCSVersion() { + try { + return CloudStackVersion.parse(getPackageVersion()); + } catch (Exception e) { + return null; + } + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java index 8016bf9c17a4..172aa16e5d72 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java @@ -101,19 +101,6 @@ private static void logRequest(HttpServletRequest req, String method, String pat String name = headerNames.nextElement(); details.append(name).append("=").append(req.getHeader(name)).append("; "); } -// String body = ""; -// if (!"GET".equalsIgnoreCase(method)) { -// StringBuilder bodySb = new StringBuilder(); -// java.io.BufferedReader reader = req.getReader(); -// if (reader != null) { -// String line; -// while ((line = reader.readLine()) != null) { -// bodySb.append(line).append('\n'); -// } -// } -// body = bodySb.toString().trim(); -// } -// details.append(", Body: ").append(body); LOGGER.debug(details.toString()); } catch (Exception e) { LOGGER.debug("Failed to capture request details", e); @@ -135,8 +122,8 @@ protected void handleRoot(HttpServletRequest req, HttpServletResponse resp, Nego } writer.write(resp, 200, Map.of( - "name", "CloudStack Veeam Control Service", - "pluginVersion", "0.1"), outFormat); + "name", VeeamControlService.PLUGIN_NAME, + "pluginVersion", this.getClass().getPackage().getImplementationVersion()), outFormat); } public void methodNotAllowed(final HttpServletResponse resp, final String allow, final Negotiation.OutFormat outFormat) throws IOException { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 706752c62819..48332b702d13 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -238,7 +238,8 @@ public class ServerAdapter extends ManagerBase { Storage.StoragePoolType.NetworkFilesystem, Storage.StoragePoolType.SharedMountPoint ); - public static final String WORKER_VM_GUEST_CPU_MODE = "host-passthrough"; + private static final String VM_TA_KEY = "veeam_tag"; + private static final String WORKER_VM_GUEST_CPU_MODE = "host-passthrough"; @Inject RoleService roleService; @@ -413,22 +414,6 @@ protected Pair getDefaultServiceAccount() { accountService.getActiveAccountById(userAccount.getAccountId())); } - protected Pair getServiceAccount() { - String serviceAccountUuid = VeeamControlService.ServiceAccountId.value(); - if (StringUtils.isEmpty(serviceAccountUuid)) { - throw new CloudRuntimeException("Service account is not configured, unable to proceed"); - } - Account account = accountService.getActiveAccountByUuid(serviceAccountUuid); - if (account == null) { - throw new CloudRuntimeException("Service account with ID " + serviceAccountUuid + " not found, unable to proceed"); - } - User user = accountService.getOneActiveUserForAccount(account); - if (user == null) { - throw new CloudRuntimeException("No active user found for service account with ID " + serviceAccountUuid); - } - return new Pair<>(user, account); - } - protected void waitForJobCompletion(long jobId) { long timeoutNanos = TimeUnit.MINUTES.toNanos(5); final long deadline = System.nanoTime() + timeoutNanos; @@ -858,6 +843,22 @@ protected Map getDetailsByInstanceId(Long instanceId) { return vmInstanceDetailsDao.listDetailsKeyPairs(instanceId, true); } + public Pair getServiceAccount() { + String serviceAccountUuid = VeeamControlService.ServiceAccountId.value(); + if (StringUtils.isEmpty(serviceAccountUuid)) { + throw new CloudRuntimeException("Service account is not configured, unable to proceed"); + } + Account account = accountService.getActiveAccountByUuid(serviceAccountUuid); + if (account == null) { + throw new CloudRuntimeException("Service account with ID " + serviceAccountUuid + " not found, unable to proceed"); + } + User user = accountService.getOneActiveUserForAccount(account); + if (user == null) { + throw new CloudRuntimeException("No active user found for service account with ID " + serviceAccountUuid); + } + return new Pair<>(user, account); + } + @Override public boolean start() { getServiceAccount(); @@ -1185,15 +1186,13 @@ public VmAction shutdownInstance(String uuid, boolean async) { @ApiAccess(command = ListTagsCmd.class) protected List listTagsByInstanceId(final long instanceId) { - List vmResourceTags = resourceTagDao.listBy(instanceId, - ResourceTag.ResourceObjectType.UserVm); + ResourceTag vmResourceTag = resourceTagDao.findByKey(instanceId, + ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY); List tags = new ArrayList<>(); - for (ResourceTag t : vmResourceTags) { - if (t instanceof ResourceTagVO) { - tags.add((ResourceTagVO)t); - continue; - } - tags.add(resourceTagDao.findById(t.getId())); + if (vmResourceTag instanceof ResourceTagVO) { + tags.add((ResourceTagVO)vmResourceTag); + } else { + tags.add(resourceTagDao.findById(vmResourceTag.getId())); } return ResourceTagVOToTagConverter.toTags(tags); } @@ -1760,8 +1759,8 @@ public List listAllTags(final Long offset, final Long limit) { List tags = new ArrayList<>(getDummyTags().values()); Filter filter = new Filter(ResourceTagVO.class, "id", true, offset, limit); Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); - List vmResourceTags = resourceTagDao.listByResourceTypeAndOwners( - ResourceTag.ResourceObjectType.UserVm, ownerDetails.first(), ownerDetails.second(), filter); + List vmResourceTags = resourceTagDao.listByResourceTypeKeyAndOwners( + ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY, ownerDetails.first(), ownerDetails.second(), filter); if (CollectionUtils.isNotEmpty(vmResourceTags)) { tags.addAll(ResourceTagVOToTagConverter.toTags(vmResourceTags)); } @@ -1775,7 +1774,8 @@ public Tag getTag(String uuid) { } Tag tag = getDummyTags().get(uuid); if (tag == null) { - ResourceTagVO resourceTagVO = resourceTagDao.findByUuid(uuid); + ResourceTagVO resourceTagVO = resourceTagDao.findByResourceTypeKeyAndValue( + ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY, uuid); accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, resourceTagVO); if (resourceTagVO != null) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java similarity index 80% rename from plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java rename to plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java index d076604515a7..be71164d672b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java @@ -21,21 +21,21 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.UUID; +import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.RouteHandler; import org.apache.cloudstack.veeam.VeeamControlService; import org.apache.cloudstack.veeam.VeeamControlServlet; +import org.apache.cloudstack.veeam.adapter.ServerAdapter; import org.apache.cloudstack.veeam.api.dto.Api; import org.apache.cloudstack.veeam.api.dto.ApiSummary; import org.apache.cloudstack.veeam.api.dto.EmptyElement; import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.ProductInfo; import org.apache.cloudstack.veeam.api.dto.Ref; -import org.apache.cloudstack.veeam.api.dto.SpecialObjects; import org.apache.cloudstack.veeam.api.dto.SummaryCount; import org.apache.cloudstack.veeam.api.dto.Version; import org.apache.cloudstack.veeam.utils.Negotiation; @@ -43,9 +43,12 @@ import com.cloud.utils.UuidUtils; import com.cloud.utils.component.ManagerBase; -public class ApiService extends ManagerBase implements RouteHandler { +public class ApiRouteHandler extends ManagerBase implements RouteHandler { public static final String BASE_ROUTE = "/api"; + @Inject + ServerAdapter serverAdapter; + @Override public boolean canHandle(String method, String path) { return getSanitizedPath(path).startsWith("/api"); @@ -63,11 +66,11 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path private void handleRootApiRequest(HttpServletRequest req, HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { io.getWriter().write(resp, HttpServletResponse.SC_OK, - createDummyApi(VeeamControlService.ContextPath.value() + BASE_ROUTE), + createApiObject(VeeamControlService.ContextPath.value() + BASE_ROUTE), outFormat); } - private static Api createDummyApi(String basePath) { + protected Api createApiObject(String basePath) { Api api = new Api(); /* ---------------- Links ---------------- */ @@ -96,30 +99,11 @@ private static Api createDummyApi(String basePath) { ProductInfo productInfo = new ProductInfo(); productInfo.setInstanceId(UuidUtils.nameUUIDFromBytes( VeeamControlService.BindAddress.value().getBytes(StandardCharsets.UTF_8)).toString()); - productInfo.name = "oVirt Engine"; - - Version version = new Version(); - version.setBuild("8"); - version.setFullVersion("4.5.8-0.master.fake.el9"); - version.setMajor("4"); - version.setMinor("5"); - version.setRevision("0"); + productInfo.name = VeeamControlService.PLUGIN_NAME; - productInfo.version = version; + productInfo.version = Version.fromPackageAndCSVersion(true); api.setProductInfo(productInfo); - /* ---------------- Special objects ---------------- */ - SpecialObjects specialObjects = new SpecialObjects(); - specialObjects.setBlankTemplate(Ref.of( - basePath + "/templates/00000000-0000-0000-0000-000000000000", - "00000000-0000-0000-0000-000000000000" - )); - specialObjects.setRootTag(Ref.of( - basePath + "/tags/00000000-0000-0000-0000-000000000000", - "00000000-0000-0000-0000-000000000000" - )); - api.setSpecialObjects(specialObjects); - /* ---------------- Summary ---------------- */ ApiSummary summary = new ApiSummary(); summary.setHosts(new SummaryCount(1, 1)); @@ -132,7 +116,7 @@ private static Api createDummyApi(String basePath) { api.setTime(System.currentTimeMillis()); /* ---------------- Users ---------------- */ - String userId = UUID.randomUUID().toString(); + String userId = serverAdapter.getServiceAccount().first().getUuid(); api.setAuthenticatedUser(Ref.of(basePath + "/users/" + userId, userId)); api.setEffectiveUser(Ref.of(basePath + "/users/" + userId, userId)); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java index 92156be5e699..4855147a333e 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java @@ -222,11 +222,6 @@ public void handle(HttpServletRequest req, HttpServletResponse resp, String path io.notFound(resp, null, outFormat); } - protected static boolean isRequestAsync(HttpServletRequest req) { - String asyncStr = req.getParameter("async"); - return Boolean.TRUE.toString().equals(asyncStr); - } - protected void handleGet(final HttpServletRequest req, final HttpServletResponse resp, Negotiation.OutFormat outFormat, VeeamControlServlet io) throws IOException { try { @@ -287,7 +282,7 @@ protected void handleUpdateById(final String id, final HttpServletRequest req, f protected void handleDeleteById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); try { VmAction vm = serverAdapter.deleteInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_OK, vm, outFormat); @@ -298,7 +293,7 @@ protected void handleDeleteById(final String id, final HttpServletRequest req, f protected void handleStartVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); try { VmAction vm = serverAdapter.startInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -309,7 +304,7 @@ protected void handleStartVmById(final String id, final HttpServletRequest req, protected void handleStopVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); try { VmAction vm = serverAdapter.stopInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -320,7 +315,7 @@ protected void handleStopVmById(final String id, final HttpServletRequest req, f protected void handleShutdownVmById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); try { VmAction vm = serverAdapter.shutdownInstance(id, async); io.getWriter().write(resp, HttpServletResponse.SC_ACCEPTED, vm, outFormat); @@ -422,7 +417,7 @@ protected void handleGetSnapshotById(final String id, final HttpServletResponse protected void handleDeleteSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); try { ResourceAction action = serverAdapter.deleteSnapshot(id, async); if (action != null) { @@ -438,7 +433,7 @@ protected void handleDeleteSnapshotById(final String id, final HttpServletReques protected void handleRestoreSnapshotById(final String id, final HttpServletRequest req, final HttpServletResponse resp, final Negotiation.OutFormat outFormat, final VeeamControlServlet io) throws IOException { - boolean async = isRequestAsync(req); + boolean async = RouteHandler.isRequestAsync(req); String data = RouteHandler.getRequestData(req, logger); try { ResourceAction response = serverAdapter.revertInstanceToSnapshot(id, async); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java index f8845804e8ef..c50f4a0ecfe6 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java @@ -34,26 +34,6 @@ public class AsyncJobJoinVOToJobConverter { - public static Job toJob(String uuid, String state, long startTime) { - Job job = new Job(); - final String basePath = VeeamControlService.ContextPath.value(); - // Fill in dummy data for now, as the AsyncJobJoinVO does not contain all the necessary information to populate a Job object. - job.setId(uuid); - job.setHref(basePath + JobsRouteHandler.BASE_ROUTE + "/" + uuid); - job.setAutoCleared(Boolean.TRUE.toString()); - job.setExternal(Boolean.TRUE.toString()); - job.setLastUpdated(System.currentTimeMillis()); - job.setStartTime(startTime); - job.setStatus(state); - if ("complete".equalsIgnoreCase(state) || "finished".equalsIgnoreCase(state)) { - job.setEndTime(System.currentTimeMillis()); - } - job.setOwner(Ref.of(basePath + "/api/users/" + uuid, uuid)); - job.setDescription("Something"); - job.setLink(Collections.emptyList()); - return job; - } - public static Job toJob(AsyncJobJoinVO vo) { Job job = new Job(); final String basePath = VeeamControlService.ContextPath.value(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java index 728d38e6c31e..2f2b40908e89 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java @@ -23,9 +23,12 @@ import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.cloudstack.veeam.api.ApiRouteHandler; import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Backup; import org.apache.cloudstack.veeam.api.dto.Disk; +import org.apache.cloudstack.veeam.api.dto.Host; +import org.apache.cloudstack.veeam.api.dto.NamedList; import org.apache.cloudstack.veeam.api.dto.Vm; import com.cloud.api.query.vo.HostJoinVO; @@ -55,6 +58,16 @@ public static Backup toBackup(final BackupVO backupVO, final Function disks = disksResolver.apply(backupVO); + backup.setDisks(NamedList.of("disks", disks)); + } return backup; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java index 7b532f26c02e..42b2233393da 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.veeam.api.dto.Link; import org.apache.cloudstack.veeam.api.dto.Ref; import org.apache.cloudstack.veeam.api.dto.Version; +import org.apache.cloudstack.veeam.api.dto.Vm; import com.cloud.api.query.vo.DataCenterJoinVO; import com.cloud.dc.ClusterVO; @@ -39,19 +40,14 @@ public class ClusterVOToClusterConverter { public static Cluster toCluster(final ClusterVO vo, final Function dataCenterResolver) { final Cluster c = new Cluster(); final String basePath = VeeamControlService.ContextPath.value(); - - // NOTE: oVirt uses UUIDs. If your ClusterVO id is numeric, generate a stable UUID: - // - Prefer: store a UUID in details table and reuse it - // - Fallback: name-based UUID from "cluster:" final String clusterId = vo.getUuid(); c.setId(clusterId); c.setHref(basePath + ClustersRouteHandler.BASE_ROUTE + "/" + clusterId); c.setName(vo.getName()); - // --- sensible defaults (match your sample) c.setBallooningEnabled("true"); - c.setBiosType("q35_ovmf"); // or "q35_secure_boot" if you want to align with VM BIOS you saw + c.setBiosType(Vm.Bios.getDefault().getType()); c.setFipsMode("disabled"); c.setFirewallType("firewalld"); c.setGlusterService("false"); @@ -64,19 +60,14 @@ public static Cluster toCluster(final ClusterVO vo, final Function oVirt-like Host. * - * @param vo HostJoinVO from listHosts (join query) */ public static Host toHost(final HostJoinVO vo) { final Host h = new Host(); @@ -49,8 +48,6 @@ public static Host toHost(final HostJoinVO vo) { final String basePath = VeeamControlService.ContextPath.value(); h.setHref(basePath + HostsRouteHandler.BASE_ROUTE + "/" + hostUuid); - // --- name / address --- - // Prefer DNS name if set; otherwise fall back to IP final String name = vo.getName() != null ? vo.getName() : ("host-" + hostUuid); h.setName(name); @@ -75,9 +72,7 @@ public static Host toHost(final HostJoinVO vo) { h.setMemory(String.valueOf(vo.getTotalMemory())); h.setMaxSchedulingMemory(String.valueOf(vo.getTotalMemory() - vo.getMemUsedCapacity())); - // --- OS / versions (optional placeholders) --- - // If you want, you can set conservative defaults to match oVirt shape. - h.setType("rhel"); + h.setType("ovirt_node"); h.setAutoNumaStatus("unknown"); h.setKdumpStatus("disabled"); h.setNumaSupported("false"); @@ -85,8 +80,6 @@ public static Host toHost(final HostJoinVO vo) { h.setUpdateAvailable("false"); - // --- links/actions --- - // Start minimal (empty). Add actions only if Veeam tries to follow them. h.setActions(null); h.setLink(Collections.emptyList()); @@ -98,13 +91,13 @@ public static List toHostList(final List vos) { } private static String mapStatus(final HostJoinVO vo) { - // CloudStack examples: - // state: Up/Down/Maintenance/Error/Disconnected - // status: Up/Down/Connecting/etc - if (vo.isInMaintenanceStates()) return "maintenance"; - if (Status.Up.equals(vo.getStatus()) && ResourceState.Enabled.equals(vo.getResourceState())) return "up"; - - // Default + if (vo.isInMaintenanceStates()) { + return "maintenance"; + } + if (Status.Up.equals(vo.getStatus()) && + ResourceState.Enabled.equals(vo.getResourceState())) { + return "up"; + } return "down"; } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java index 114311225d33..82198997e7db 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java @@ -55,7 +55,6 @@ public static Network toNetwork(final NetworkVO vo, final Function oVirt datacenter ref if (dcResolver != null) { final DataCenterJoinVO dc = dcResolver.apply(vo.getDataCenterId()); if (dc != null) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java index b9d660f1fa60..af10d586c89a 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java @@ -45,7 +45,6 @@ public static VnicProfile toVnicProfile(final NetworkVO vo, final Function oVirt datacenter ref if (dcResolver != null) { final DataCenterJoinVO dc = dcResolver.apply(vo.getDataCenterId()); if (dc != null) { diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java index 165dbd1db581..b55201327ea2 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java @@ -39,6 +39,8 @@ import com.cloud.vm.NicVO; public class NicVOToNicConverter { + private static final String DEFAULT_INTERFACE_TYPE = "virtio"; + private static final String DEFAULT_REPORTED_DEVICE_NAME = "eth0"; public static Nic toNic(final NicVO vo, final String vmUuid, final Function networkResolver) { final String basePath = VeeamControlService.ContextPath.value(); @@ -56,7 +58,7 @@ public static Nic toNic(final NicVO vo, final String vmUuid, final Function oVirt-like Vm DTO. * - * @param src UserVmJoinVO */ public static Vm toVm(final UserVmJoinVO src, final Function hostResolver, @@ -84,7 +83,7 @@ public static Vm toVm(final UserVmJoinVO src, dst.setStartTime(lastUpdated.getTime()); } final Ref template = buildRef( - basePath + ApiService.BASE_ROUTE, + basePath + ApiRouteHandler.BASE_ROUTE, "templates", src.getTemplateUuid() ); @@ -92,20 +91,19 @@ public static Vm toVm(final UserVmJoinVO src, dst.setOriginalTemplate(template); if (StringUtils.isNotBlank(src.getHostUuid())) { dst.setHost(buildRef( - basePath + ApiService.BASE_ROUTE, + basePath + ApiRouteHandler.BASE_ROUTE, "hosts", src.getHostUuid())); - } if (hostResolver != null) { HostJoinVO hostVo = hostResolver.apply(src.getHostId() == null ? src.getLastHostId() : src.getHostId()); if (hostVo != null) { dst.setHost(buildRef( - basePath + ApiService.BASE_ROUTE, + basePath + ApiRouteHandler.BASE_ROUTE, "hosts", hostVo.getUuid())); dst.setCluster(buildRef( - basePath + ApiService.BASE_ROUTE, + basePath + ApiRouteHandler.BASE_ROUTE, "clusters", hostVo.getClusterUuid())); } @@ -123,9 +121,7 @@ public static Vm toVm(final UserVmJoinVO src, cpu.setTopology(new Topology(src.getCpu(), 1, 1)); dst.setCpu(cpu); Os os = new Os(); - os.setType(src.getGuestOsId() % 2 == 0 - ? "windows" - : "linux"); + os.setType(src.getGuestOsDisplayName()); Os.Boot boot = new Os.Boot(); boot.setDevices(NamedList.of("device", List.of("hd"))); os.setBoot(boot); @@ -167,7 +163,7 @@ public static Vm toVm(final UserVmJoinVO src, dst.setTags(NamedList.of("tag", tags)); } dst.setCpuProfile(Ref.of( - basePath + ApiService.BASE_ROUTE + "/cpuprofiles/" + src.getServiceOfferingUuid(), + basePath + ApiRouteHandler.BASE_ROUTE + "/cpuprofiles/" + src.getServiceOfferingUuid(), src.getServiceOfferingUuid())); if (allContent) { dst.setInitialization(getOvfInitialization(dst, src)); @@ -204,9 +200,10 @@ public static List toVmList(final List srcList, } private static String mapStatus(final VirtualMachine.State state) { - // CloudStack-ish states -> oVirt-ish up/down - if (Arrays.asList(VirtualMachine.State.Running, - VirtualMachine.State.Migrating, VirtualMachine.State.Restoring).contains(state)) { + if (Arrays.asList( + VirtualMachine.State.Running, + VirtualMachine.State.Migrating, + VirtualMachine.State.Restoring).contains(state)) { return "up"; } return "down"; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java index b1be9b988042..af92e7a10f22 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java @@ -24,7 +24,7 @@ import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.veeam.VeeamControlService; -import org.apache.cloudstack.veeam.api.ApiService; +import org.apache.cloudstack.veeam.api.ApiRouteHandler; import org.apache.cloudstack.veeam.api.DisksRouteHandler; import org.apache.cloudstack.veeam.api.VmsRouteHandler; import org.apache.cloudstack.veeam.api.dto.Disk; @@ -43,7 +43,7 @@ public class VolumeJoinVOToDiskConverter { public static Disk toDisk(final VolumeJoinVO vol, final Function physicalSizeResolver) { final Disk disk = new Disk(); final String basePath = VeeamControlService.ContextPath.value(); - final String apiBasePath = basePath + ApiService.BASE_ROUTE; + final String apiBasePath = basePath + ApiRouteHandler.BASE_ROUTE; final String diskId = vol.getUuid(); final String diskHref = basePath + DisksRouteHandler.BASE_ROUTE + "/" + diskId; diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java index 6d612fa38ebe..b337541bf5ca 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java @@ -23,9 +23,11 @@ public class Backup extends BaseDto { private String description; private Long creationDate; private Vm vm; + private Host host; private String phase; private String fromCheckpointId; private String toCheckpointId; + private NamedList disks; public String getName() { return name; @@ -59,6 +61,14 @@ public void setVm(Vm vm) { this.vm = vm; } + public Host getHost() { + return host; + } + + public void setHost(Host host) { + this.host = host; + } + public String getPhase() { return phase; } @@ -82,4 +92,12 @@ public String getToCheckpointId() { public void setToCheckpointId(String toCheckpointId) { this.toCheckpointId = toCheckpointId; } + + public NamedList getDisks() { + return disks; + } + + public void setDisks(NamedList disks) { + this.disks = disks; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java index 5f98ca775dc2..0b260a5cdcd7 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java @@ -46,4 +46,10 @@ public void setId(String id) { public static Link getActionLink(final String action, final String baseHref) { return Link.of(action, baseHref + "/" + action); } + + protected static T withHrefAndId(T dto, String href, String id) { + dto.setHref(href); + dto.setId(id); + return dto; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java index 8c4dba1d57c6..73efba5eeb89 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java @@ -308,4 +308,8 @@ public void setVersion(String version) { this.version = version; } } + + public static Host of(String href, String id) { + return withHrefAndId(new Host(), href, id); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index 667eb7d00b11..7b7d80a0f16c 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -17,6 +17,10 @@ package org.apache.cloudstack.veeam.api.dto; +import org.apache.cloudstack.utils.CloudStackVersion; +import org.apache.cloudstack.veeam.VeeamControlService; +import org.apache.commons.lang3.StringUtils; + import com.fasterxml.jackson.annotation.JsonInclude; @JsonInclude(JsonInclude.Include.NON_NULL) @@ -70,4 +74,21 @@ public String getRevision() { public void setRevision(String revision) { this.revision = revision; } + + public static Version fromPackageAndCSVersion(boolean complete) { + Version version = new Version(); + String packageVersion = VeeamControlService.getPackageVersion(); + if (StringUtils.isNotBlank(packageVersion) && complete) { + version.setFullVersion(packageVersion); + } + CloudStackVersion csVersion = VeeamControlService.getCSVersion(); + if (csVersion == null) { + return version; + } + version.setMajor(String.valueOf(csVersion.getMajorRelease())); + version.setMinor(String.valueOf(csVersion.getMinorRelease())); + version.setBuild(String.valueOf(csVersion.getPatchRelease())); + version.setRevision(String.valueOf(csVersion.getSecurityRelease())); + return version; + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 90a50207aacc..9607e7949981 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -513,9 +513,6 @@ public void setType(String type) { } public static Vm of(String href, String id) { - Vm vm = new Vm(); - vm.setHref(href); - vm.setId(id); - return vm; + return withHrefAndId(new Vm(), href, id); } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java deleted file mode 100644 index fa67367773eb..000000000000 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/response/FaultResponse.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.cloudstack.veeam.api.response; - -import org.apache.cloudstack.veeam.api.dto.Fault; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JacksonXmlRootElement(localName = "fault") -public final class FaultResponse { - public Fault fault; - - public FaultResponse() {} - - public FaultResponse(final Fault fault) { - this.fault = fault; - } - - public static FaultResponse of(final String reason, final String detail) { - return new FaultResponse(new Fault(reason, detail)); - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java index b69748bf8bd0..8fe2a48c7029 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java @@ -25,6 +25,7 @@ import com.cloud.utils.UuidUtils; public class PathUtil { + private static final boolean CONSIDER_ONLY_UUID_AS_ID = false; public static List extractIdAndSubPath(final String path, final String baseRoute) { @@ -65,7 +66,7 @@ public static List extractIdAndSubPath(final String path, final String b } // Validate first segment is a UUID - if (validParts.isEmpty() || !UuidUtils.isUuid(validParts.get(0))) { + if (validParts.isEmpty() || (CONSIDER_ONLY_UUID_AS_ID && !UuidUtils.isUuid(validParts.get(0)))) { return null; } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java index 4b191c6c3adc..51d2f829f3db 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java @@ -23,7 +23,6 @@ import javax.servlet.http.HttpServletResponse; import org.apache.cloudstack.veeam.api.dto.Fault; -import org.apache.cloudstack.veeam.api.response.FaultResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,7 +77,7 @@ public void writeFault(final HttpServletResponse resp, final int status, final S if (fmt == Negotiation.OutFormat.XML) { write(resp, status, fault, fmt); } else { - write(resp, status, new FaultResponse(fault), fmt); + write(resp, status, fault, fmt); } } } diff --git a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml index 4d66d5248e04..83d100ec76e8 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml @@ -36,7 +36,7 @@ - + diff --git a/plugins/integrations/veeam-control-service/src/main/resources/test.xml b/plugins/integrations/veeam-control-service/src/main/resources/test-ovf.xml similarity index 92% rename from plugins/integrations/veeam-control-service/src/main/resources/test.xml rename to plugins/integrations/veeam-control-service/src/main/resources/test-ovf.xml index 5af3b9be4353..53688f0b82ef 100644 --- a/plugins/integrations/veeam-control-service/src/main/resources/test.xml +++ b/plugins/integrations/veeam-control-service/src/main/resources/test-ovf.xml @@ -1,3 +1,21 @@ + Date: Thu, 9 Apr 2026 23:52:05 +0530 Subject: [PATCH 111/129] merge fixes Signed-off-by: Abhishek Kumar --- .../api/command/admin/backup/CreateImageTransferCmd.java | 2 +- plugins/integrations/veeam-control-service/pom.xml | 2 +- .../org/apache/cloudstack/veeam/adapter/ServerAdapter.java | 4 ++-- .../main/java/org/apache/cloudstack/veeam/api/dto/Vm.java | 2 +- .../network/contrail/management/MockAccountManager.java | 4 ++++ server/src/main/java/com/cloud/user/AccountManager.java | 2 ++ server/src/main/java/com/cloud/user/AccountManagerImpl.java | 6 ++++++ 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index 8948d1a0d5fe..eeb63b985d5c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -81,7 +81,7 @@ public ImageTransfer.Direction getDirection() { } public ImageTransfer.Format getFormat() { - return EnumUtils.fromString(ImageTransfer.Format.class, format); + return EnumUtils.getEnum(ImageTransfer.Format.class, format); } @Override diff --git a/plugins/integrations/veeam-control-service/pom.xml b/plugins/integrations/veeam-control-service/pom.xml index cc0349b75d60..4b1b1f4501aa 100644 --- a/plugins/integrations/veeam-control-service/pom.xml +++ b/plugins/integrations/veeam-control-service/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.22.1.0-SNAPSHOT + 4.23.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 48332b702d13..3990b1e129ac 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1447,11 +1447,11 @@ public ImageTransfer createImageTransfer(ImageTransfer request) { } accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, volumeVO); - Direction direction = EnumUtils.fromString(Direction.class, request.getDirection()); + Direction direction = EnumUtils.getEnum(Direction.class, request.getDirection()); if (direction == null) { throw new InvalidParameterValueException("Invalid or missing direction"); } - Format format = EnumUtils.fromString(Format.class, request.getFormat()); + Format format = EnumUtils.getEnum(Format.class, request.getFormat()); Long backupId = null; if (request.getBackup() != null && StringUtils.isNotBlank(request.getBackup().getId())) { BackupVO backupVO = backupDao.findByUuid(request.getBackup().getId()); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java index 9607e7949981..1d557d186f08 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java @@ -348,7 +348,7 @@ public String getType() { @JsonIgnore public int getTypeOrdinal() { - Type enumType = EnumUtils.fromString(Type.class, type, Type.q35_sea_bios); + Type enumType = EnumUtils.getEnum(Type.class, type, Type.q35_sea_bios); return enumType.ordinal(); } diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java index ab7662f44309..4ec966362355 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java @@ -597,6 +597,10 @@ public void validateUserPasswordAndUpdateIfNeeded(String newPassword, UserVO use public void checkApiAccess(Account account, String command, String apiKey) throws PermissionDeniedException { } + @Override + public void checkApiAccess(Account caller, String command) throws PermissionDeniedException { + } + @Override public UserAccount clearUserTwoFactorAuthenticationInSetupStateOnLogin(UserAccount user) { return null; diff --git a/server/src/main/java/com/cloud/user/AccountManager.java b/server/src/main/java/com/cloud/user/AccountManager.java index 98d2419e0486..eca1a571dd88 100644 --- a/server/src/main/java/com/cloud/user/AccountManager.java +++ b/server/src/main/java/com/cloud/user/AccountManager.java @@ -204,6 +204,8 @@ void buildACLViewSearchCriteria(SearchCriteria s void checkApiAccess(Account caller, String command, String apiKey); + void checkApiAccess(Account caller, String command); + UserAccount clearUserTwoFactorAuthenticationInSetupStateOnLogin(UserAccount user); void verifyCallerPrivilegeForUserOrAccountOperations(Account userAccount); diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index c9f4feea8e90..9c7c8141f8e0 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -1535,6 +1535,12 @@ public void checkApiAccess(Account caller, String command, String apiKey) { checkApiAccess(apiCheckers, caller, command, keyPairPermissions.toArray(new ApiKeyPairPermission[0])); } + @Override + public void checkApiAccess(Account caller, String command) { + List apiCheckers = getEnabledApiCheckers(); + checkApiAccess(apiCheckers, caller, command); + } + @NotNull private List getEnabledApiCheckers() { // we are really only interested in the dynamic access checker From 605f7bff3f114db1b474cf67301d87c8b2aad8ed Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 10 Apr 2026 07:49:27 +0530 Subject: [PATCH 112/129] Add stress test. Fix concurrency. --- .../vm/hypervisor/kvm/imageserver/__init__.py | 7 +- .../hypervisor/kvm/imageserver/concurrency.py | 7 +- .../hypervisor/kvm/imageserver/constants.py | 4 +- .../vm/hypervisor/kvm/imageserver/handler.py | 36 -- .../kvm/imageserver/tests/__init__.py | 6 + .../kvm/imageserver/tests/test_stress_io.py | 414 ++++++++++++++++++ 6 files changed, 425 insertions(+), 49 deletions(-) create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py diff --git a/scripts/vm/hypervisor/kvm/imageserver/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/__init__.py index dc9505310395..7392dfd3b756 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/__init__.py +++ b/scripts/vm/hypervisor/kvm/imageserver/__init__.py @@ -28,10 +28,5 @@ - file: read/write a local qcow2/raw file; full PUT only, GET with optional ranges, flush. -Usage:: - - # As a module - python -m imageserver --listen 127.0.0.1 --port 54322 - - # Or via the systemd service started by createImageTransfer +Run as a systemd service by the CreateImageTransfer CloudStack Agent Command """ diff --git a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py index 7d91aea60131..6b2d28a4069b 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py +++ b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py @@ -38,8 +38,8 @@ class ConcurrencyManager: """ def __init__(self, max_reads: int, max_writes: int): - self._max_reads = max_reads - self._max_writes = max_writes + self._max_reads = max_reads + 4 + self._max_writes = max_writes + 4 self._images: Dict[str, _ImageState] = {} self._guard = threading.Lock() @@ -66,6 +66,3 @@ def acquire_write(self, image_id: str, blocking: bool = False) -> bool: def release_write(self, image_id: str) -> None: self._state_for(image_id).write_sem.release() - - def get_image_lock(self, image_id: str) -> threading.Lock: - return self._state_for(image_id).lock diff --git a/scripts/vm/hypervisor/kvm/imageserver/constants.py b/scripts/vm/hypervisor/kvm/imageserver/constants.py index 0b6465527f4b..4f5bfd1a7372 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/constants.py +++ b/scripts/vm/hypervisor/kvm/imageserver/constants.py @@ -23,8 +23,8 @@ # NBD qemu:dirty-bitmap flags (dirty=1) NBD_STATE_DIRTY = 1 -MAX_PARALLEL_READS = 8 -MAX_PARALLEL_WRITES = 1 +MAX_PARALLEL_READS = 4 +MAX_PARALLEL_WRITES = 4 # HTTP server defaults DEFAULT_LISTEN_ADDRESS = "127.0.0.1" diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index c28a06575814..9775e7049f9b 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -570,11 +570,7 @@ def _handle_get_image( def _handle_put_image( self, image_id: str, cfg: Dict[str, Any], content_length: int, flush: bool ) -> None: - lock = self._concurrency.get_image_lock(image_id) - lock.acquire() - if not self._concurrency.acquire_write(image_id): - lock.release() self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") return @@ -598,7 +594,6 @@ def _handle_put_image( self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: self._concurrency.release_write(image_id) - lock.release() dur = now_s() - start logging.info( "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur @@ -612,11 +607,7 @@ def _handle_put_range( content_length: int, flush: bool, ) -> None: - lock = self._concurrency.get_image_lock(image_id) - lock.acquire() - if not self._concurrency.acquire_write(image_id): - lock.release() self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") return @@ -657,7 +648,6 @@ def _handle_put_range( self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: self._concurrency.release_write(image_id) - lock.release() dur = now_s() - start logging.info( "PUT range end image_id=%s bytes=%d duration_s=%.3f flush=%s", @@ -667,11 +657,6 @@ def _handle_put_range( def _handle_get_extents( self, image_id: str, cfg: Dict[str, Any], context: Optional[str] = None ) -> None: - lock = self._concurrency.get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - start = now_s() try: logging.info("EXTENTS start image_id=%s context=%s", image_id, context) @@ -709,16 +694,10 @@ def _handle_get_extents( logging.error("EXTENTS error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - lock.release() dur = now_s() - start logging.info("EXTENTS end image_id=%s duration_s=%.3f", image_id, dur) def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: - lock = self._concurrency.get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - start = now_s() try: logging.info("FLUSH start image_id=%s", image_id) @@ -732,7 +711,6 @@ def _handle_post_flush(self, image_id: str, cfg: Dict[str, Any]) -> None: logging.error("FLUSH error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - lock.release() dur = now_s() - start logging.info("FLUSH end image_id=%s duration_s=%.3f", image_id, dur) @@ -744,13 +722,7 @@ def _handle_patch_zero( size: int, flush: bool, ) -> None: - lock = self._concurrency.get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - if not self._concurrency.acquire_write(image_id): - lock.release() self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") return @@ -775,7 +747,6 @@ def _handle_patch_zero( self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: self._concurrency.release_write(image_id) - lock.release() dur = now_s() - start logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) @@ -786,13 +757,7 @@ def _handle_patch_range( range_header: str, content_length: int, ) -> None: - lock = self._concurrency.get_image_lock(image_id) - if not lock.acquire(blocking=False): - self._send_error_json(HTTPStatus.CONFLICT, "image busy") - return - if not self._concurrency.acquire_write(image_id): - lock.release() self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") return @@ -840,7 +805,6 @@ def _handle_patch_range( self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: self._concurrency.release_write(image_id) - lock.release() dur = now_s() - start logging.info( "PATCH range end image_id=%s bytes=%d duration_s=%.3f", diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py index 0ccbeeeafb7c..09102f9da2bb 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py @@ -14,3 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +""" +Run: +cd to the directory containing the imageserver folder +python3 -m unittest discover -s imageserver/tests -t . -v +""" \ No newline at end of file diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py new file mode 100644 index 000000000000..87b10726344b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py @@ -0,0 +1,414 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Stress IO tests +They run only when IMAGESERVER_STRESS_TEST_QCOW_DIR is set to an existing +directory containing qcow2 files. +""" + +import json +import os +import subprocess +import time +import unittest +import uuid +import urllib.error +from concurrent.futures import ThreadPoolExecutor, as_completed + +from imageserver.constants import MAX_PARALLEL_READS, MAX_PARALLEL_WRITES + +from .test_base import get_tmp_dir, http_get, http_post, http_put, make_nbd_transfer_existing_disk + + +def _allocated_subranges(extents, granularity): + """Split each non-hole extent (zero=False) into [start, end] inclusive byte ranges.""" + out = [] + for ext in extents: + if ext.get("zero"): + continue + start = int(ext["start"]) + length = int(ext["length"]) + pos = start + end_abs = start + length + while pos < end_abs: + chunk_end = min(pos + granularity, end_abs) + out.append((pos, chunk_end - 1)) + pos = chunk_end + return out + + +def _qemu_img_virtual_size(path: str) -> int: + """Return virtual size in bytes (requires ``qemu-img`` on PATH).""" + cp = subprocess.run( + ["qemu-img", "info", "--output=json", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + return int(json.loads(cp.stdout)["virtual-size"]) + + +def _http_error_detail(exc: urllib.error.HTTPError) -> str: + parts = ["HTTP %s %r" % (exc.code, exc.reason), "url=%r" % getattr(exc, "url", "")] + try: + if exc.fp is not None: + raw = exc.fp.read() + if raw: + text = raw.decode("utf-8", errors="replace") + parts.append("response_body=%r" % (text,)) + except Exception as read_err: + parts.append("read_body_error=%r" % (read_err,)) + return "; ".join(parts) + + +def _http_get_checked(url, headers=None, expected_status=200, label="GET"): + try: + resp = http_get(url, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError("%s failed for %r: %s" % (label, url, _http_error_detail(e))) from e + if resp.status != expected_status: + body = resp.read() + raise AssertionError( + "%s %r: expected HTTP %s, got %s; body=%r" + % (label, url, expected_status, resp.status, body) + ) + return resp + + +def _http_put_checked(url, data, headers, label="PUT"): + try: + resp = http_put(url, data, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError("%s failed for %r: %s" % (label, url, _http_error_detail(e))) from e + body = resp.read() + if resp.status != 200: + raise AssertionError( + "%s %r: expected HTTP 200, got %s; body=%r" % (label, url, resp.status, body) + ) + return resp, body + + +def _http_post_checked(url, data=b"", headers=None, label="POST"): + try: + resp = http_post(url, data=data, headers=headers) + except urllib.error.HTTPError as e: + raise AssertionError("%s failed for %r: %s" % (label, url, _http_error_detail(e))) from e + body = resp.read() + if resp.status != 200: + raise AssertionError( + "%s %r: expected HTTP 200, got %s; body=%r" % (label, url, resp.status, body) + ) + return resp, body + + +def _list_qcow2_files(dir_path: str): + entries = [] + for name in os.listdir(dir_path): + p = os.path.join(dir_path, name) + if not os.path.isfile(p): + continue + # Keep this intentionally permissive; qemu-nbd can still reject invalid files. + if name.lower().endswith(".qcow2") or name.lower().endswith(".qcow"): + entries.append(p) + entries.sort() + return entries + + +class TestQcow2ExtentsParallelReads(unittest.TestCase): + """ + For each qcow2 in IMAGESERVER_STRESS_TEST_QCOW_DIR, + export it via qemu-nbd, fetch allocation extents, and perform parallel range reads + over allocated regions. A second test copies allocated extents into a new qcow2 + and validates via qemu-img compare. + + Env: + - IMAGESERVER_STRESS_TEST_QCOW_DIR: directory containing qcow2 files (required) + - IMAGESERVER_STRESS_TEST_READ_GRANULARITY: byte step (default 4 MiB) + (fallback: IMAGESERVER_TEST_QCOW2_READ_GRANULARITY for compatibility) + """ + + def setUp(self): + super().setUp() + self._qcow_dir = os.environ.get("IMAGESERVER_STRESS_TEST_QCOW_DIR", "").strip() + if not self._qcow_dir or not os.path.isdir(self._qcow_dir): + self.skipTest( + "Set IMAGESERVER_STRESS_TEST_QCOW_DIR to an existing directory containing qcow2 files" + ) + + self._dest_dir = self._qcow_dir.rstrip(os.sep) + ".test" + try: + os.makedirs(self._dest_dir, exist_ok=True) + except OSError as e: + self.skipTest("failed to create dest dir %r: %r" % (self._dest_dir, e)) + + raw_g = os.environ.get("IMAGESERVER_STRESS_TEST_READ_GRANULARITY", "").strip() + if not raw_g: + raw_g = os.environ.get("IMAGESERVER_TEST_QCOW2_READ_GRANULARITY", "").strip() + self._read_granularity = int(raw_g) if raw_g else 4 * 1024 * 1024 + if self._read_granularity <= 0: + self.skipTest("IMAGESERVER_STRESS_TEST_READ_GRANULARITY must be positive") + + self._qcow2_files = _list_qcow2_files(self._qcow_dir) + if not self._qcow2_files: + self.skipTest("no qcow2 files found in IMAGESERVER_STRESS_TEST_QCOW_DIR") + + # Avoid pathological oversubscription by default; still runs multiple files concurrently. + cpu = os.cpu_count() or 4 + self._file_workers = max(1, min(len(self._qcow2_files), cpu)) + + def test_parallel_range_reads_allocated_extents(self): + """ + For every qcow2 in the directory: GET /extents then do parallel Range GETs across + allocated spans. All qcow2 files are processed concurrently. + """ + + def run_one(path: str): + _, url, server, cleanup = make_nbd_transfer_existing_disk(path, "qcow2") + try: + resp = _http_get_checked( + "%s/extents" % (url,), + expected_status=200, + label="GET /extents", + ) + extents = json.loads(resp.read()) + ranges = _allocated_subranges(extents, self._read_granularity) + if not ranges: + # Not an error; some images can legitimately be all holes. + return {"path": path, "ranges": 0, "skipped": True} + + def fetch(span): + start_b, end_b = span + range_hdr = "bytes=%s-%s" % (start_b, end_b) + r = _http_get_checked( + url, + headers={"Range": range_hdr}, + expected_status=206, + label="Range GET %s" % (range_hdr,), + ) + data = r.read() + expected_len = end_b - start_b + 1 + if len(data) != expected_len: + raise AssertionError( + "Range GET %s: got %d bytes, expected %d (url=%r, file=%r)" + % (range_hdr, len(data), expected_len, url, path) + ) + + with ThreadPoolExecutor(max_workers=MAX_PARALLEL_READS) as pool: + list(pool.map(fetch, ranges)) + return {"path": path, "ranges": len(ranges), "skipped": False} + finally: + cleanup() + + started = time.perf_counter() + results = [] + failures = [] + with ThreadPoolExecutor(max_workers=self._file_workers) as pool: + futs = {pool.submit(run_one, p): p for p in self._qcow2_files} + for fut in as_completed(futs): + p = futs[fut] + try: + results.append(fut.result()) + except Exception as e: + failures.append((p, e)) + + elapsed = time.perf_counter() - started + skipped = sum(1 for r in results if r.get("skipped")) + total_ranges = sum(int(r.get("ranges", 0)) for r in results) + print( + "stress_io: test_parallel_range_reads_allocated_extents: files=%d workers=%d skipped=%d total_ranges=%d elapsed=%.3fs" + % (len(self._qcow2_files), self._file_workers, skipped, total_ranges, elapsed) + ) + + if failures: + first_path, first_exc = failures[0] + raise AssertionError( + "stress_io: %d/%d files failed (first=%r): %r" + % (len(failures), len(self._qcow2_files), first_path, first_exc) + ) from first_exc + + def test_parallel_reads_then_put_range_copy_matches_source(self): + """ + For every qcow2 in the directory: create an empty qcow2 with same virtual size, + then copy every allocated range using a worker pool (Range GET then Content-Range PUT), + flush, and validate via qemu-img compare. All qcow2 files are processed concurrently. + """ + + def run_one(src_path: str): + try: + vsize = _qemu_img_virtual_size(src_path) + except ( + FileNotFoundError, + subprocess.CalledProcessError, + KeyError, + json.JSONDecodeError, + TypeError, + ValueError, + ) as e: + raise AssertionError("qemu-img info failed for %r: %r" % (src_path, e)) from e + + base = os.path.basename(src_path) + # Keep dest names unique in case the same basename appears more than once. + dest_name = "%s.copy.%s.qcow2" % (base, uuid.uuid4().hex[:8]) + dest_path = os.path.join(self._dest_dir, dest_name) + try: + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", dest_path, str(vsize)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + raise AssertionError("qemu-img create failed for %r: %r" % (dest_path, e)) from e + + _, src_url, _, cleanup_src = make_nbd_transfer_existing_disk(src_path, "qcow2") + _, dest_url, _, cleanup_dest = make_nbd_transfer_existing_disk(dest_path, "qcow2") + try: + resp = _http_get_checked( + "%s/extents" % (src_url,), + expected_status=200, + label="GET src /extents", + ) + extents = json.loads(resp.read()) + ranges = _allocated_subranges(extents, self._read_granularity) + if not ranges: + return {"path": src_path, "ranges": 0, "skipped": True} + + transfer_workers = max(1, min(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES)) + + def transfer_span(span): + start_b, end_b = span + range_hdr = "bytes=%s-%s" % (start_b, end_b) + r = _http_get_checked( + src_url, + headers={"Range": range_hdr}, + expected_status=206, + label="Range GET src %s" % (range_hdr,), + ) + data = r.read() + expected_len = end_b - start_b + 1 + if len(data) != expected_len: + raise AssertionError( + "Range GET src %s: got %d bytes, expected %d (url=%r, file=%r)" + % (range_hdr, len(data), expected_len, src_url, src_path) + ) + end_inclusive = start_b + len(data) - 1 + cr = "bytes %s-%s/*" % (start_b, end_inclusive) + _put_resp, put_body = _http_put_checked( + dest_url, + data, + headers={ + "Content-Range": cr, + "Content-Length": str(len(data)), + }, + label="PUT dest %s" % (cr,), + ) + try: + body = json.loads(put_body) + except ValueError: + raise AssertionError( + "PUT dest %s: invalid JSON body=%r (url=%r, file=%r)" + % (cr, put_body, dest_url, src_path) + ) + if not body.get("ok"): + raise AssertionError( + "PUT dest %s: JSON ok=false, full=%r (url=%r, file=%r)" + % (cr, body, dest_url, src_path) + ) + if body.get("bytes_written") != len(data): + raise AssertionError( + "PUT dest %s: bytes_written=%r expected %d (url=%r, file=%r)" + % (cr, body.get("bytes_written"), len(data), dest_url, src_path) + ) + + with ThreadPoolExecutor(max_workers=transfer_workers) as pool: + list(pool.map(transfer_span, ranges)) + + _flush, flush_body = _http_post_checked( + "%s/flush" % (dest_url,), + label="POST dest /flush", + ) + try: + flush_json = json.loads(flush_body) + except ValueError as e: + raise AssertionError( + "POST dest /flush: invalid JSON body=%r (url=%r, file=%r)" + % (flush_body, dest_url, src_path) + ) from e + if not flush_json.get("ok"): + raise AssertionError( + "POST dest /flush: ok=false, full=%r (url=%r, file=%r)" + % (flush_json, dest_url, src_path) + ) + finally: + cleanup_dest() + cleanup_src() + + try: + cmp = subprocess.run( + ["qemu-img", "compare", src_path, dest_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + if cmp.returncode != 0: + raise AssertionError( + "qemu-img compare %r vs %r failed (rc=%s): stderr=%r stdout=%r" + % (src_path, dest_path, cmp.returncode, cmp.stderr, cmp.stdout) + ) + finally: + try: + os.unlink(dest_path) + except FileNotFoundError: + pass + + return {"path": src_path, "ranges": len(ranges), "skipped": False} + + started = time.perf_counter() + results = [] + failures = [] + with ThreadPoolExecutor(max_workers=self._file_workers) as pool: + futs = {pool.submit(run_one, p): p for p in self._qcow2_files} + for fut in as_completed(futs): + p = futs[fut] + try: + results.append(fut.result()) + except Exception as e: + failures.append((p, e)) + + elapsed = time.perf_counter() - started + skipped = sum(1 for r in results if r.get("skipped")) + total_ranges = sum(int(r.get("ranges", 0)) for r in results) + print( + "stress_io: test_parallel_reads_then_put_range_copy_matches_source: files=%d workers=%d skipped=%d total_ranges=%d elapsed=%.3fs" + % (len(self._qcow2_files), self._file_workers, skipped, total_ranges, elapsed) + ) + + if failures: + first_path, first_exc = failures[0] + raise AssertionError( + "stress_io: %d/%d files failed (first=%r): %r" + % (len(failures), len(self._qcow2_files), first_path, first_exc) + ) from first_exc + + +if __name__ == "__main__": + unittest.main() + + From ef1a47ea6a0ba93345f7d7ec27583b4fa8b1f332 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 10 Apr 2026 08:00:43 +0530 Subject: [PATCH 113/129] max writers as 1 for file backend --- scripts/vm/hypervisor/kvm/imageserver/handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index 9775e7049f9b..4701a7581a97 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -216,6 +216,10 @@ def do_OPTIONS(self) -> None: with self._registry.request_lifecycle(image_id): backend = create_backend(cfg) try: + max_writers = MAX_PARALLEL_WRITES + if not backend.supports_range_write: + max_writers = 1 + if not backend.supports_extents: allowed_methods = "GET, PUT, POST, OPTIONS" features = ["flush"] @@ -223,7 +227,7 @@ def do_OPTIONS(self) -> None: "unix_socket": None, "features": features, "max_readers": MAX_PARALLEL_READS, - "max_writers": MAX_PARALLEL_WRITES, + "max_writers": max_writers, } self._send_json(HTTPStatus.OK, response, allowed_methods=allowed_methods) return @@ -254,7 +258,6 @@ def do_OPTIONS(self) -> None: features.append("zero") if can_flush: features.append("flush") - max_writers = MAX_PARALLEL_WRITES response = { "unix_socket": None, From 527db66f8c183038c6a2bd7784b0d2643cab25b4 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:02:02 +0530 Subject: [PATCH 114/129] fix precommit --- scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py | 2 +- scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py index 09102f9da2bb..f8d0ff6d0061 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py @@ -19,4 +19,4 @@ Run: cd to the directory containing the imageserver folder python3 -m unittest discover -s imageserver/tests -t . -v -""" \ No newline at end of file +""" diff --git a/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py b/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py index 87b10726344b..39b24ebf39f2 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py +++ b/scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py @@ -410,5 +410,3 @@ def transfer_span(span): if __name__ == "__main__": unittest.main() - - From 411122b97c466b4f7509f4f868b1da74f1942251 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Sun, 12 Apr 2026 22:43:46 +0530 Subject: [PATCH 115/129] Fix fd double free in nbd backend --- .../vm/hypervisor/kvm/imageserver/backends/nbd.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py index aa247be29f21..c68a3b5188b6 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py +++ b/scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py @@ -45,8 +45,6 @@ def __init__( need_block_status: bool = False, extra_meta_contexts: Optional[List[str]] = None, ): - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._sock.connect(socket_path) self._nbd = nbd.NBD() if export and hasattr(self._nbd, "set_export_name"): @@ -59,7 +57,7 @@ def __init__( except Exception as e: logging.warning("add_meta_context %r failed: %r", ctx, e) - self._connect_existing_socket(self._sock) + self._nbd.connect_unix(socket_path) def _connect_existing_socket(self, sock: socket.socket) -> None: last_err: Optional[BaseException] = None @@ -308,15 +306,6 @@ def close(self) -> None: self._nbd.shutdown() except Exception: pass - try: - if hasattr(self._nbd, "close"): - self._nbd.close() - except Exception: - pass - try: - self._sock.close() - except Exception: - pass def __enter__(self) -> "NbdConnection": return self From ff12afb8a4a9fbae0563fa8fb84aea560a29725e Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Mon, 13 Apr 2026 00:06:14 +0530 Subject: [PATCH 116/129] don't allow image transfer creation if image transfer entry is already there. --- .../apache/cloudstack/backup/KVMBackupExportServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 3b160ce4885a..57697ffbddd6 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -618,9 +618,9 @@ public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTran " backup provider. Either set backup.framework.enabled to false or set the Zone level config backup.framework.provider.plugin to \"dummy\"."); } - ImageTransferVO existingTransfer = imageTransferDao.findUnfinishedByVolume(volume.getId()); + ImageTransferVO existingTransfer = imageTransferDao.findByVolume(volume.getId()); if (existingTransfer != null) { - throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid()); + throw new CloudRuntimeException("Image transfer already exists for volume: " + volume.getUuid()); } ImageTransfer.Backend backend = getImageTransferBackend(format, direction); From 1d20ecc677ab37d97544c65c140cac8258f60d95 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Mon, 13 Apr 2026 00:06:27 +0530 Subject: [PATCH 117/129] fix image transfer response object name --- .../api/command/admin/backup/CreateImageTransferCmd.java | 1 + .../api/command/admin/backup/FinalizeImageTransferCmd.java | 2 ++ .../api/command/admin/backup/ListImageTransfersCmd.java | 2 ++ 3 files changed, 5 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java index eeb63b985d5c..cc6992afd88d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -87,6 +87,7 @@ public ImageTransfer.Format getFormat() { @Override public void execute() { ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java index d483f78b4228..dbbe18ed280d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; @@ -56,6 +57,7 @@ public void execute() { boolean result = kvmBackupExportService.finalizeImageTransfer(this); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java index 2565ef241a6b..d810d21ab5f8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.ImageTransferResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.backup.KVMBackupExportService; import org.apache.cloudstack.context.CallContext; @@ -68,6 +69,7 @@ public void execute() { List responses = kvmBackupExportService.listImageTransfers(this); ListResponse response = new ListResponse<>(); response.setResponses(responses); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); response.setResponseName(getCommandName()); setResponseObject(response); } From 9af2c941aed22a04b384fc273c7ce76ebb63525a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha Date: Mon, 13 Apr 2026 00:31:08 +0530 Subject: [PATCH 118/129] rename image_transfer disk_id to volume_id --- .../cloudstack/backup/ImageTransfer.java | 2 +- .../cloudstack/backup/ImageTransferVO.java | 24 +++++++++---------- .../backup/dao/ImageTransferDaoImpl.java | 4 ++-- ...ageTransferVOToImageTransferConverter.java | 2 +- .../backup/KVMBackupExportServiceImpl.java | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java index f7fe1e9c2bb2..e1153be3ae02 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -45,7 +45,7 @@ public enum Phase { Long getBackupId(); - long getDiskId(); + long getVolumeId(); long getHostId(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index c391eae2e86b..7c8af972bbe2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -45,8 +45,8 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "backup_id") private Long backupId; - @Column(name = "disk_id") - private long diskId; + @Column(name = "volume_id") + private long volumeId; @Column(name = "host_id") private long hostId; @@ -102,9 +102,9 @@ public class ImageTransferVO implements ImageTransfer { public ImageTransferVO() { } - private ImageTransferVO(String uuid, long diskId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + private ImageTransferVO(String uuid, long volumeId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { this.uuid = uuid; - this.diskId = diskId; + this.volumeId = volumeId; this.hostId = hostId; this.phase = phase; this.direction = direction; @@ -114,15 +114,15 @@ private ImageTransferVO(String uuid, long diskId, long hostId, Phase phase, Dire this.created = new Date(); } - public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { - this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId); + public ImageTransferVO(String uuid, Long backupId, long volumeId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); this.backupId = backupId; this.socket = socket; this.backend = Backend.nbd; } - public ImageTransferVO(String uuid, long diskId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { - this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId); + public ImageTransferVO(String uuid, long volumeId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); this.file = file; this.backend = Backend.file; } @@ -147,12 +147,12 @@ public void setBackupId(long backupId) { } @Override - public long getDiskId() { - return diskId; + public long getVolumeId() { + return volumeId; } - public void setDiskId(long diskId) { - this.diskId = diskId; + public void setVolumeId(long volumeId) { + this.volumeId = volumeId; } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java index 3e1f6b513a58..0448180fd6aa 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -54,11 +54,11 @@ protected void init() { uuidSearch.done(); volumeSearch = createSearchBuilder(); - volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); volumeSearch.done(); volumeUnfinishedSearch = createSearchBuilder(); - volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getDiskId(), SearchCriteria.Op.EQ); + volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ); volumeUnfinishedSearch.done(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java index 084f644d317f..98b0baa1e13b 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java @@ -64,7 +64,7 @@ public static ImageTransfer toImageTransfer(ImageTransferVO vo, final Function volumeSizes = new HashMap<>(); for (ImageTransferVO transfer : hostTransfers) { - VolumeVO volume = volumeDao.findById(transfer.getDiskId()); + VolumeVO volume = volumeDao.findById(transfer.getVolumeId()); if (volume == null) { logger.warn("Volume not found for image transfer: {}", transfer.getUuid()); imageTransferDao.remove(transfer.getId()); // ToDo: confirm if this enough? From a9fb479805cf6ab5da957fd8912de89600188d60 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 13 Apr 2026 00:44:18 +0530 Subject: [PATCH 119/129] Use executor service for pollImageTransferProgress --- .../backup/KVMBackupExportServiceImpl.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index a4d4502ca6bb..e9de55a20d37 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -27,9 +27,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.inject.Inject; @@ -52,7 +53,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.jobs.JobInfo; -import org.apache.cloudstack.managed.context.ManagedContextTimerTask; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.MapUtils; @@ -83,6 +84,7 @@ import com.cloud.user.User; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.ReflectionUse; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -140,7 +142,7 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject AsyncJobManager asyncJobManager; - private Timer imageTransferTimer; + private ScheduledExecutorService imageTransferStatusExecutor; VmWorkJobHandlerProxy jobHandlerProxy = new VmWorkJobHandlerProxy(this); @@ -884,7 +886,9 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans @Override public boolean start() { - final TimerTask imageTransferPollTask = new ManagedContextTimerTask() { + imageTransferStatusExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Image-Transfer-Status-Executor")); + long pollingInterval = ImageTransferPollingInterval.value(); + imageTransferStatusExecutor.scheduleAtFixedRate(new ManagedContextRunnable() { @Override protected void runInContext() { try { @@ -893,20 +897,13 @@ protected void runInContext() { logger.warn("Catch throwable in image transfer poll task ", t); } } - }; - - imageTransferTimer = new Timer("ImageTransferPollTask"); - long pollingInterval = ImageTransferPollingInterval.value() * 1000L; - imageTransferTimer.schedule(imageTransferPollTask, pollingInterval, pollingInterval); + }, pollingInterval, pollingInterval, TimeUnit.SECONDS); return true; } @Override public boolean stop() { - if (imageTransferTimer != null) { - imageTransferTimer.cancel(); - imageTransferTimer = null; - } + imageTransferStatusExecutor.shutdown(); return true; } @@ -973,7 +970,7 @@ private void pollImageTransferProgress() { VolumeVO volume = volumeDao.findById(transfer.getVolumeId()); if (volume == null) { logger.warn("Volume not found for image transfer: {}", transfer.getUuid()); - imageTransferDao.remove(transfer.getId()); // ToDo: confirm if this enough? + imageTransferDao.remove(transfer.getId()); continue; } transferVolumeMap.put(transfer.getId(), volume); @@ -1000,8 +997,9 @@ private void pollImageTransferProgress() { if (answer == null || !answer.getResult() || MapUtils.isEmpty(answer.getProgressMap())) { logger.warn("Failed to get progress for transfers on host {}: {}", hostId, answer != null ? answer.getDetails() : "null answer"); - return; // ToDo: return on continue? + continue; } + for (ImageTransferVO transfer : hostTransfers) { String transferId = transfer.getUuid(); Long currentSize = answer.getProgressMap().get(transferId); From e2aac4110b1fd0228c5e3e55720087822803416e Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 13 Apr 2026 01:09:34 +0530 Subject: [PATCH 120/129] Add package dependency for python3-libnbd and socat --- debian/control | 2 +- packaging/el8/cloud.spec | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 2b8ce929c639..cdf663ef8906 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 705959336f15..ee3151e2210a 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -125,6 +125,8 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: python3-libnbd +Requires: socat Provides: cloud-agent Group: System Environment/Libraries %description agent From e5fd64b835343bcf78760b1ba3a491d0bbf4519c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Apr 2026 13:02:07 +0530 Subject: [PATCH 121/129] refactor tags Signed-off-by: Abhishek Kumar --- .../com/cloud/tags/dao/ResourceTagDao.java | 10 +++-- .../cloud/tags/dao/ResourceTagsDaoImpl.java | 39 +++++++++++++------ .../veeam/adapter/ServerAdapter.java | 16 +++----- .../ResourceTagVOToTagConverter.java | 15 +++++++ 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index 034ea61ee0e9..5efaea40a943 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -63,8 +63,12 @@ public interface ResourceTagDao extends GenericDao { List listByResourceUuid(String resourceUuid); - List listByResourceTypeKeyAndOwners(ResourceObjectType resourceType, String key, - List accountIds, List domainIds, Filter filter); + List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter); + + ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, String value); + + List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, String key); - ResourceTagVO findByResourceTypeKeyAndValue(ResourceObjectType resourceType, String key, String value); } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index b82dd5ec3dec..22c7b7b2ee5b 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -31,6 +31,7 @@ import com.cloud.tags.ResourceTagVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -124,12 +125,13 @@ public List listByResourceUuid(String resourceUuid) { } @Override - public List listByResourceTypeKeyAndOwners(ResourceObjectType resourceType, String key, - List accountIds, List domainIds, - Filter filter) { - SearchBuilder sb = createSearchBuilder(); + public List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter) { + GenericSearchBuilder sb = createSearchBuilder(String.class); + sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getValue()); sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); - sb.and("key", sb.entity().getKey(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); if (accountIdsNotEmpty || domainIdsNotEmpty) { @@ -138,30 +140,45 @@ public List listByResourceTypeKeyAndOwners(ResourceObjectType res sb.cp(); } sb.done(); - final SearchCriteria sc = sb.create(); + final SearchCriteria sc = sb.create(); sc.setParameters("resourceType", resourceType); - sc.setParameters("key", key); + sc.setParameters("key", key + "%"); if (accountIdsNotEmpty) { sc.setParameters("account", accountIds.toArray()); } if (domainIdsNotEmpty) { sc.setParameters("domain", domainIds.toArray()); } - return listBy(sc, filter); + return customSearch(sc, filter); } @Override - public ResourceTagVO findByResourceTypeKeyAndValue(ResourceObjectType resourceType, String key, + public ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, String value) { SearchBuilder sb = createSearchBuilder(); sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); - sb.and("key", sb.entity().getKey(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); sb.and("value", sb.entity().getValue(), Op.EQ); sb.done(); final SearchCriteria sc = sb.create(); sc.setParameters("resourceType", resourceType); - sc.setParameters("key", key); + sc.setParameters("key", key + "%"); sc.setParameters("value", value); return findOneBy(sc); } + + @Override + public List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, + String key) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("resourceId", resourceId); + sc.setParameters("key", key + "%"); + return listBy(sc); + } } diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java index 3990b1e129ac..4b07f32ee03d 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java @@ -1186,14 +1186,8 @@ public VmAction shutdownInstance(String uuid, boolean async) { @ApiAccess(command = ListTagsCmd.class) protected List listTagsByInstanceId(final long instanceId) { - ResourceTag vmResourceTag = resourceTagDao.findByKey(instanceId, - ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY); - List tags = new ArrayList<>(); - if (vmResourceTag instanceof ResourceTagVO) { - tags.add((ResourceTagVO)vmResourceTag); - } else { - tags.add(resourceTagDao.findById(vmResourceTag.getId())); - } + List tags = resourceTagDao.listByResourceTypeIdAndKeyPrefix( + ResourceTag.ResourceObjectType.UserVm, instanceId, VM_TA_KEY); return ResourceTagVOToTagConverter.toTags(tags); } @@ -1759,10 +1753,10 @@ public List listAllTags(final Long offset, final Long limit) { List tags = new ArrayList<>(getDummyTags().values()); Filter filter = new Filter(ResourceTagVO.class, "id", true, offset, limit); Pair, List> ownerDetails = getResourceOwnerFiltersWithDomainIds(); - List vmResourceTags = resourceTagDao.listByResourceTypeKeyAndOwners( + List vmResourceTags = resourceTagDao.listByResourceTypeKeyPrefixAndOwners( ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY, ownerDetails.first(), ownerDetails.second(), filter); if (CollectionUtils.isNotEmpty(vmResourceTags)) { - tags.addAll(ResourceTagVOToTagConverter.toTags(vmResourceTags)); + tags.addAll(ResourceTagVOToTagConverter.toTagsFromValues(vmResourceTags)); } return tags; } @@ -1774,7 +1768,7 @@ public Tag getTag(String uuid) { } Tag tag = getDummyTags().get(uuid); if (tag == null) { - ResourceTagVO resourceTagVO = resourceTagDao.findByResourceTypeKeyAndValue( + ResourceTagVO resourceTagVO = resourceTagDao.findByResourceTypeKeyPrefixAndValue( ResourceTag.ResourceObjectType.UserVm, VM_TA_KEY, uuid); accountService.checkAccess(CallContext.current().getCallingAccount(), null, false, resourceTagVO); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java index 9715b0321103..38d67a778378 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java @@ -61,7 +61,22 @@ public static Tag toTag(ResourceTagVO vo) { return tag; } + public static Tag toTag(String id) { + String basePath = VeeamControlService.ContextPath.value(); + Tag tag = new Tag(); + tag.setId(id); + tag.setName(id); + tag.setDescription(String.format("Tag: %s", id)); + tag.setHref(basePath + TagsRouteHandler.BASE_ROUTE + "/" + id); + tag.setParent(getRootTagRef()); + return tag; + } + public static List toTags(List vos) { return vos.stream().map(ResourceTagVOToTagConverter::toTag).collect(Collectors.toList()); } + + public static List toTagsFromValues(List values) { + return values.stream().map(ResourceTagVOToTagConverter::toTag).collect(Collectors.toList()); + } } From c40b30bc4a02e0661619ce98c0a1e51d14ddec53 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 14 Apr 2026 21:20:41 +0530 Subject: [PATCH 122/129] Remove ImagetransferProgress Command --- .../api/response/ImageTransferResponse.java | 8 -- .../backup/KVMBackupExportService.java | 5 - .../GetImageTransferProgressAnswer.java | 47 -------- .../GetImageTransferProgressCommand.java | 67 ----------- .../cloudstack/backup/ImageTransferVO.java | 12 -- ...etImageTransferProgressCommandWrapper.java | 95 --------------- ...ageTransferVOToImageTransferConverter.java | 2 +- .../backup/KVMBackupExportServiceImpl.java | 108 ------------------ 8 files changed, 1 insertion(+), 343 deletions(-) delete mode 100644 core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java delete mode 100644 core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java delete mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java index 8a24ed3966f2..15576e8f1012 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -62,10 +62,6 @@ public class ImageTransferResponse extends BaseResponse { @Param(description = "the image transfer direction: upload / download") private String direction; - @SerializedName("progress") - @Param(description = "progress in percentage for the upload image transfer") - private Integer progress; - @SerializedName(ApiConstants.CREATED) @Param(description = "the date created") private Date created; @@ -102,10 +98,6 @@ public void setDirection(String direction) { this.direction = direction; } - public void setProgress(Integer progress) { - this.progress = progress; - } - public void setCreated(Date created) { this.created = created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index 51e52c85ec34..a40b2b5b5eda 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -38,11 +38,6 @@ */ public interface KVMBackupExportService extends Configurable, PluggableService { - ConfigKey ImageTransferPollingInterval = new ConfigKey<>("Advanced", Long.class, - "image.transfer.polling.interval", - "10", - "The image transfer progress polling interval in seconds.", true, ConfigKey.Scope.Global); - ConfigKey ImageTransferIdleTimeoutSeconds = new ConfigKey<>("Advanced", Integer.class, "image.transfer.idle.timeout.seconds", "600", diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java deleted file mode 100644 index 5b5713f4683a..000000000000 --- a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java +++ /dev/null @@ -1,47 +0,0 @@ -//Licensed to the Apache Software Foundation (ASF) under one -//or more contributor license agreements. See the NOTICE file -//distributed with this work for additional information -//regarding copyright ownership. The ASF licenses this file -//to you under the Apache License, Version 2.0 (the -//"License"); you may not use this file except in compliance -//the License. You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, -//software distributed under the License is distributed on an -//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -//KIND, either express or implied. See the License for the -//specific language governing permissions and limitations -//under the License. - -package org.apache.cloudstack.backup; - -import java.util.Map; - -import com.cloud.agent.api.Answer; - -public class GetImageTransferProgressAnswer extends Answer { - private Map progressMap; // transferId -> progress percentage (0-100) - - public GetImageTransferProgressAnswer() { - } - - public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details) { - super(cmd, success, details); - } - - public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details, - Map progressMap) { - super(cmd, success, details); - this.progressMap = progressMap; - } - - public Map getProgressMap() { - return progressMap; - } - - public void setProgressMap(Map progressMap) { - this.progressMap = progressMap; - } -} diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java deleted file mode 100644 index 2391f957f51f..000000000000 --- a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressCommand.java +++ /dev/null @@ -1,67 +0,0 @@ -//Licensed to the Apache Software Foundation (ASF) under one -//or more contributor license agreements. See the NOTICE file -//distributed with this work for additional information -//regarding copyright ownership. The ASF licenses this file -//to you under the Apache License, Version 2.0 (the -//"License"); you may not use this file except in compliance -//the License. You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, -//software distributed under the License is distributed on an -//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -//KIND, either express or implied. See the License for the -//specific language governing permissions and limitations -//under the License. - -package org.apache.cloudstack.backup; - -import java.util.List; -import java.util.Map; - -import com.cloud.agent.api.Command; - -public class GetImageTransferProgressCommand extends Command { - private List transferIds; - private Map volumePaths; // transferId -> volume path - private Map volumeSizes; // transferId -> volume size - - public GetImageTransferProgressCommand() { - } - - public GetImageTransferProgressCommand(List transferIds, Map volumePaths, Map volumeSizes) { - this.transferIds = transferIds; - this.volumePaths = volumePaths; - this.volumeSizes = volumeSizes; - } - - public List getTransferIds() { - return transferIds; - } - - public void setTransferIds(List transferIds) { - this.transferIds = transferIds; - } - - public Map getVolumePaths() { - return volumePaths; - } - - public void setVolumePaths(Map volumePaths) { - this.volumePaths = volumePaths; - } - - public Map getVolumeSizes() { - return volumeSizes; - } - - public void setVolumeSizes(Map volumeSizes) { - this.volumeSizes = volumeSizes; - } - - @Override - public boolean executeInSequence() { - return false; - } -} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index 7c8af972bbe2..ec9b927b63e4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -75,9 +75,6 @@ public class ImageTransferVO implements ImageTransfer { @Column(name = "signed_ticket_id") private String signedTicketId; - @Column(name = "progress") - private Integer progress; - @Column(name = "account_id") Long accountId; @@ -210,15 +207,6 @@ public void setSignedTicketId(String signedTicketId) { this.signedTicketId = signedTicketId; } - public Integer getProgress() { - return progress; - } - - public void setProgress(Integer progress) { - this.progress = progress; - this.updated = new Date(); - } - @Override public Class getEntityType() { return ImageTransfer.class; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java deleted file mode 100644 index 7e0cbf2934db..000000000000 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java +++ /dev/null @@ -1,95 +0,0 @@ -//Licensed to the Apache Software Foundation (ASF) under one -//or more contributor license agreements. See the NOTICE file -//distributed with this work for additional information -//regarding copyright ownership. The ASF licenses this file -//to you under the Apache License, Version 2.0 (the -//"License"); you may not use this file except in compliance -//the License. You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, -//software distributed under the License is distributed on an -//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -//KIND, either express or implied. See the License for the -//specific language governing permissions and limitations -//under the License. - -package com.cloud.hypervisor.kvm.resource.wrapper; - -import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.backup.GetImageTransferProgressAnswer; -import org.apache.cloudstack.backup.GetImageTransferProgressCommand; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.cloud.agent.api.Answer; -import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; -import com.cloud.resource.CommandWrapper; -import com.cloud.resource.ResourceWrapper; - -@ResourceWrapper(handles = GetImageTransferProgressCommand.class) -public class LibvirtGetImageTransferProgressCommandWrapper extends CommandWrapper { - protected Logger logger = LogManager.getLogger(getClass()); - - @Override - public Answer execute(GetImageTransferProgressCommand cmd, LibvirtComputingResource resource) { - try { - List transferIds = cmd.getTransferIds(); - Map volumePaths = cmd.getVolumePaths(); - Map volumeSizes = cmd.getVolumeSizes(); - Map progressMap = new HashMap<>(); - - if (transferIds == null || transferIds.isEmpty()) { - return new GetImageTransferProgressAnswer(cmd, true, "No transfers to check", progressMap); - } - - for (String transferId : transferIds) { - String volumePath = volumePaths.get(transferId); - Long volumeSize = volumeSizes.get(transferId); - - if (volumePath == null || volumeSize == null || volumeSize == 0) { - logger.warn("Missing volume path or size for transferId: {}", transferId); - progressMap.put(transferId, null); - continue; - } - - try { - File file = new File(volumePath); - if (!file.exists()) { - logger.warn("Volume file does not exist: {}", volumePath); - progressMap.put(transferId, null); - continue; - } - - long currentSize = file.length(); - - if (volumePath.endsWith(".qcow2") || volumePath.endsWith(".qcow")) { - try { - currentSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); - } catch (Exception e) { - logger.warn("Failed to get virtual size for qcow2 file: {}, using physical size", volumePath, e); - } - } - progressMap.put(transferId, currentSize); - logger.debug("Transfer {} progress, current: {})", transferId, currentSize, volumeSize); - - } catch (Exception e) { - logger.error("Error getting progress for transferId: {}, path: {}", transferId, volumePath, e); - progressMap.put(transferId, null); - } - } - - return new GetImageTransferProgressAnswer(cmd, true, "Progress retrieved successfully", progressMap); - - } catch (Exception e) { - logger.error("Error executing GetImageTransferProgressCommand", e); - return new GetImageTransferProgressAnswer(cmd, false, "Error getting transfer progress: " + e.getMessage()); - } - } -} diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java index 98b0baa1e13b..3be1865895a8 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java @@ -42,7 +42,7 @@ public static ImageTransfer toImageTransfer(ImageTransferVO vo, final Function 0 && vo.getProgress() < 100)); + imageTransfer.setActive(Boolean.toString(org.apache.cloudstack.backup.ImageTransfer.Phase.transferring.equals(vo.getPhase()))); imageTransfer.setDirection(vo.getDirection().name()); imageTransfer.setFormat("cow"); imageTransfer.setInactivityTimeout(Integer.toString(3600)); diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index e9de55a20d37..57a094531443 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -28,9 +28,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.inject.Inject; @@ -53,10 +50,8 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.jobs.JobInfo; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; @@ -84,7 +79,6 @@ import com.cloud.user.User; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.ReflectionUse; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -142,8 +136,6 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject AsyncJobManager asyncJobManager; - private ScheduledExecutorService imageTransferStatusExecutor; - VmWorkJobHandlerProxy jobHandlerProxy = new VmWorkJobHandlerProxy(this); private boolean isKVMBackupExportServiceSupported(Long zoneId) { @@ -878,7 +870,6 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans response.setDiskId(volume.getUuid()); response.setTransferUrl(imageTransferVO.getTransferUrl()); response.setPhase(imageTransferVO.getPhase().toString()); - response.setProgress(imageTransferVO.getProgress()); response.setDirection(imageTransferVO.getDirection().toString()); response.setCreated(imageTransferVO.getCreated()); return response; @@ -886,24 +877,11 @@ private ImageTransferResponse toImageTransferResponse(ImageTransferVO imageTrans @Override public boolean start() { - imageTransferStatusExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Image-Transfer-Status-Executor")); - long pollingInterval = ImageTransferPollingInterval.value(); - imageTransferStatusExecutor.scheduleAtFixedRate(new ManagedContextRunnable() { - @Override - protected void runInContext() { - try { - pollImageTransferProgress(); - } catch (final Throwable t) { - logger.warn("Catch throwable in image transfer poll task ", t); - } - } - }, pollingInterval, pollingInterval, TimeUnit.SECONDS); return true; } @Override public boolean stop() { - imageTransferStatusExecutor.shutdown(); return true; } @@ -945,91 +923,6 @@ protected Pair waitForBackupTerminalState(final long bac } } - private void pollImageTransferProgress() { - try { - List transferringTransfers = imageTransferDao.listByPhaseAndDirection( - ImageTransfer.Phase.transferring, ImageTransfer.Direction.upload); - if (transferringTransfers == null || transferringTransfers.isEmpty()) { - return; - } - - Map> transfersByHost = transferringTransfers.stream() - .collect(Collectors.groupingBy(ImageTransferVO::getHostId)); - Map transferVolumeMap = new HashMap<>(); - - for (Map.Entry> entry : transfersByHost.entrySet()) { - Long hostId = entry.getKey(); - List hostTransfers = entry.getValue(); - - try { - List transferIds = new ArrayList<>(); - Map volumePaths = new HashMap<>(); - Map volumeSizes = new HashMap<>(); - - for (ImageTransferVO transfer : hostTransfers) { - VolumeVO volume = volumeDao.findById(transfer.getVolumeId()); - if (volume == null) { - logger.warn("Volume not found for image transfer: {}", transfer.getUuid()); - imageTransferDao.remove(transfer.getId()); - continue; - } - transferVolumeMap.put(transfer.getId(), volume); - - String transferId = transfer.getUuid(); - transferIds.add(transferId); - - if (volume.getPath() == null) { - logger.warn("Volume path is null for image transfer: {}", transfer.getUuid()); - continue; - } - String volumePath = getVolumePathForFileBasedBackend(volume); - volumePaths.put(transferId, volumePath); - volumeSizes.put(transferId, volume.getSize()); - } - - if (transferIds.isEmpty()) { - continue; - } - - GetImageTransferProgressCommand cmd = new GetImageTransferProgressCommand(transferIds, volumePaths, volumeSizes); - GetImageTransferProgressAnswer answer = (GetImageTransferProgressAnswer) agentManager.send(hostId, cmd); - - if (answer == null || !answer.getResult() || MapUtils.isEmpty(answer.getProgressMap())) { - logger.warn("Failed to get progress for transfers on host {}: {}", hostId, - answer != null ? answer.getDetails() : "null answer"); - continue; - } - - for (ImageTransferVO transfer : hostTransfers) { - String transferId = transfer.getUuid(); - Long currentSize = answer.getProgressMap().get(transferId); - if (currentSize == null) { - continue; - } - VolumeVO volume = transferVolumeMap.get(transfer.getId()); - long totalSize = getVolumeTotalSize(volume); - int progress = Math.max((int)((currentSize * 100) / totalSize), 100); - transfer.setProgress(progress); - if (currentSize >= 100) { - transfer.setPhase(ImageTransfer.Phase.finished); - logger.debug("Updated phase for image transfer {} to finished", transferId); - } - imageTransferDao.update(transfer.getId(), transfer); - logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); - } - - } catch (AgentUnavailableException | OperationTimedoutException e) { - logger.warn("Failed to communicate with host {} for image transfer progress", hostId); - } catch (Exception e) { - logger.error("Error polling image transfer progress for host " + hostId, e); - } - } - - } catch (Exception e) { - logger.error("Error in pollImageTransferProgress", e); - } - } - private long getVolumeTotalSize(VolumeVO volume) { VolumeDetailVO detail = volumeDetailsDao.findDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE); if (detail != null) { @@ -1063,7 +956,6 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - ImageTransferPollingInterval, ImageTransferIdleTimeoutSeconds, ExposeKVMBackupExportServiceApis }; From 00d1dbc3639b89094f77f14268a790a013ded5e5 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 14 Apr 2026 21:51:54 +0530 Subject: [PATCH 123/129] Remove image server per-image concurrency locks --- .../hypervisor/kvm/imageserver/concurrency.py | 68 ------------------- .../vm/hypervisor/kvm/imageserver/handler.py | 29 +------- .../vm/hypervisor/kvm/imageserver/server.py | 8 +-- 3 files changed, 2 insertions(+), 103 deletions(-) delete mode 100644 scripts/vm/hypervisor/kvm/imageserver/concurrency.py diff --git a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py b/scripts/vm/hypervisor/kvm/imageserver/concurrency.py deleted file mode 100644 index 6b2d28a4069b..000000000000 --- a/scripts/vm/hypervisor/kvm/imageserver/concurrency.py +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import threading -from typing import Dict, NamedTuple - - -class _ImageState(NamedTuple): - read_sem: threading.Semaphore - write_sem: threading.Semaphore - lock: threading.Lock - - -class ConcurrencyManager: - """ - Manages per-image read/write semaphores and per-image mutual-exclusion locks. - - Each image_id gets its own independent pool of read slots (default MAX_PARALLEL_READS) - and write slots (default MAX_PARALLEL_WRITES), so concurrent transfers to different images - do not contend with each other. - - The per-image lock serialises operations that must not overlap on the - same image (e.g. flush while writing, extents while writing). - """ - - def __init__(self, max_reads: int, max_writes: int): - self._max_reads = max_reads + 4 - self._max_writes = max_writes + 4 - self._images: Dict[str, _ImageState] = {} - self._guard = threading.Lock() - - def _state_for(self, image_id: str) -> _ImageState: - with self._guard: - state = self._images.get(image_id) - if state is None: - state = _ImageState( - read_sem=threading.Semaphore(self._max_reads), - write_sem=threading.Semaphore(self._max_writes), - lock=threading.Lock(), - ) - self._images[image_id] = state - return state - - def acquire_read(self, image_id: str, blocking: bool = False) -> bool: - return self._state_for(image_id).read_sem.acquire(blocking=blocking) - - def release_read(self, image_id: str) -> None: - self._state_for(image_id).read_sem.release() - - def acquire_write(self, image_id: str, blocking: bool = False) -> bool: - return self._state_for(image_id).write_sem.acquire(blocking=blocking) - - def release_write(self, image_id: str) -> None: - self._state_for(image_id).write_sem.release() diff --git a/scripts/vm/hypervisor/kvm/imageserver/handler.py b/scripts/vm/hypervisor/kvm/imageserver/handler.py index 4701a7581a97..d2d97d7810b6 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/handler.py +++ b/scripts/vm/hypervisor/kvm/imageserver/handler.py @@ -24,7 +24,6 @@ from urllib.parse import parse_qs from .backends import NbdBackend, create_backend -from .concurrency import ConcurrencyManager from .config import TransferRegistry from .constants import CHUNK_SIZE, MAX_PARALLEL_READS, MAX_PARALLEL_WRITES, MAX_PATCH_JSON_SIZE from .util import is_fallback_dirty_response, json_bytes, now_s @@ -38,14 +37,13 @@ class Handler(BaseHTTPRequestHandler): All backend I/O is delegated to ImageBackend implementations via the create_backend() factory. - Class-level attributes _concurrency and _registry are injected + Class-level attribute _registry is injected by the server at startup (see server.py / make_handler()). """ server_version = "cloudstack-image-server/1.0" server_protocol = "HTTP/1.1" - _concurrency: ConcurrencyManager _registry: TransferRegistry _CONTENT_RANGE_RE = re.compile(r"^bytes\s+(\d+)-(\d+)/(?:\*|\d+)$") @@ -493,10 +491,6 @@ def do_PATCH(self) -> None: def _handle_get_image( self, image_id: str, cfg: Dict[str, Any], range_header: Optional[str] ) -> None: - if not self._concurrency.acquire_read(image_id): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel reads") - return - start = now_s() bytes_sent = 0 try: @@ -564,7 +558,6 @@ def _handle_get_image( except Exception: pass finally: - self._concurrency.release_read(image_id) dur = now_s() - start logging.info( "GET end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_sent, dur @@ -573,10 +566,6 @@ def _handle_get_image( def _handle_put_image( self, image_id: str, cfg: Dict[str, Any], content_length: int, flush: bool ) -> None: - if not self._concurrency.acquire_write(image_id): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - start = now_s() bytes_written = 0 try: @@ -596,7 +585,6 @@ def _handle_put_image( logging.error("PUT error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - self._concurrency.release_write(image_id) dur = now_s() - start logging.info( "PUT end image_id=%s bytes=%d duration_s=%.3f", image_id, bytes_written, dur @@ -610,10 +598,6 @@ def _handle_put_range( content_length: int, flush: bool, ) -> None: - if not self._concurrency.acquire_write(image_id): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - start = now_s() bytes_written = 0 try: @@ -650,7 +634,6 @@ def _handle_put_range( logging.error("PUT range error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - self._concurrency.release_write(image_id) dur = now_s() - start logging.info( "PUT range end image_id=%s bytes=%d duration_s=%.3f flush=%s", @@ -725,10 +708,6 @@ def _handle_patch_zero( size: int, flush: bool, ) -> None: - if not self._concurrency.acquire_write(image_id): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - start = now_s() try: logging.info( @@ -749,7 +728,6 @@ def _handle_patch_zero( logging.error("PATCH zero error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - self._concurrency.release_write(image_id) dur = now_s() - start logging.info("PATCH zero end image_id=%s duration_s=%.3f", image_id, dur) @@ -760,10 +738,6 @@ def _handle_patch_range( range_header: str, content_length: int, ) -> None: - if not self._concurrency.acquire_write(image_id): - self._send_error_json(HTTPStatus.SERVICE_UNAVAILABLE, "too many parallel writes") - return - start = now_s() bytes_written = 0 try: @@ -807,7 +781,6 @@ def _handle_patch_range( logging.error("PATCH range error image_id=%s err=%r", image_id, e) self._send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "backend error") finally: - self._concurrency.release_write(image_id) dur = now_s() - start logging.info( "PATCH range end image_id=%s bytes=%d duration_s=%.3f", diff --git a/scripts/vm/hypervisor/kvm/imageserver/server.py b/scripts/vm/hypervisor/kvm/imageserver/server.py index 1bc42252d4f2..50bd4e0b1391 100644 --- a/scripts/vm/hypervisor/kvm/imageserver/server.py +++ b/scripts/vm/hypervisor/kvm/imageserver/server.py @@ -33,7 +33,6 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] pass -from .concurrency import ConcurrencyManager from .config import TransferRegistry, validate_transfer_config from .constants import ( CONTROL_RECV_BUFFER, @@ -42,14 +41,11 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): # type: ignore[no-redef] CONTROL_SOCKET_PERMISSIONS, DEFAULT_HTTP_PORT, DEFAULT_LISTEN_ADDRESS, - MAX_PARALLEL_READS, - MAX_PARALLEL_WRITES, ) from .handler import Handler def make_handler( - concurrency: ConcurrencyManager, registry: TransferRegistry, ) -> Type[Handler]: """ @@ -60,7 +56,6 @@ def make_handler( """ class ConfiguredHandler(Handler): - _concurrency = concurrency _registry = registry return ConfiguredHandler @@ -186,8 +181,7 @@ def main() -> None: ) registry = TransferRegistry() - concurrency = ConcurrencyManager(MAX_PARALLEL_READS, MAX_PARALLEL_WRITES) - handler_cls = make_handler(concurrency, registry) + handler_cls = make_handler(registry) ctrl_thread = threading.Thread( target=_control_listener, From fb82ca3c918b36692fd0d3f21aa1a73ccd0ba862 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2025 17:20:26 +0530 Subject: [PATCH 124/129] config: fix ManagementServer scope Signed-off-by: Abhishek Kumar --- .../command/admin/config/ListCfgsByCmd.java | 25 ++++-- .../api/command/admin/config/ResetCfgCmd.java | 15 ++++ .../command/admin/config/UpdateCfgCmd.java | 16 +++- ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42210to42300.sql | 12 +++ framework/cluster/pom.xml | 6 ++ .../cluster/ManagementServerHostDetailVO.java | 87 +++++++++++++++++++ .../dao/ManagementServerHostDetailsDao.java | 27 ++++++ .../ManagementServerHostDetailsDaoImpl.java | 46 ++++++++++ .../ConfigurationManagerImpl.java | 34 +++++++- .../cloud/server/ManagementServerImpl.java | 6 ++ ui/src/components/view/SettingsTab.vue | 4 + .../config/section/infra/managementServers.js | 4 + 13 files changed, 275 insertions(+), 8 deletions(-) create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index f6f66415f533..a7757cf0ee38 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -19,23 +19,24 @@ import java.util.ArrayList; import java.util.List; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.commons.lang3.StringUtils; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; @@ -94,6 +95,13 @@ public class ListCfgsByCmd extends BaseListCmd { description = "The ID of the Image Store to update the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "Lists configuration by group name (primarily used for UI)", since = "4.18.0") private String groupName; @@ -139,6 +147,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + public String getGroupName() { return groupName; } @@ -200,6 +212,9 @@ private void setScope(ConfigurationResponse cfgResponse) { if (getImageStoreId() != null){ cfgResponse.setScope("imagestore"); } + if (getManagementServerId() != null){ + cfgResponse.setScope("managementserver"); + } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java index 2d511cff34db..5e7d38c830f7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.api.response.AccountResponse; @@ -84,6 +85,13 @@ public class ResetCfgCmd extends BaseCmd { description = "The ID of the Image Store to reset the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -116,6 +124,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -149,6 +161,9 @@ public void execute() { if (getImageStoreId() != null) { response.setScope(ConfigKey.Scope.ImageStore.name()); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name()); + } response.setValue(cfg.second()); this.setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index 97dee8f638af..c6fb62b4ff8a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; @@ -88,6 +89,13 @@ public class UpdateCfgCmd extends BaseCmd { validations = ApiArgValidator.PositiveNumber) private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,7 +120,7 @@ public Long getClusterId() { return clusterId; } - public Long getStoragepoolId() { + public Long getStoragePoolId() { return storagePoolId; } @@ -128,6 +136,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -182,7 +194,7 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getClusterId() != null) { response.setScope("cluster"); } - if (getStoragepoolId() != null) { + if (getStoragePoolId() != null) { response.setScope("storagepool"); } if (getAccountId() != null) { diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index fda874745dfa..0e72337ec451 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -116,6 +116,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 47b28964acdc..9e928fe0c77c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -118,6 +118,18 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); +-- Add management_server_details table to allow ManagementServer scope configs +CREATE TABLE IF NOT EXISTS `management_server_details` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `management_server_id` bigint unsigned NOT NULL COMMENT 'management server the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_management_server_details__management_server_id` FOREIGN KEY `fk_management_server_details__management_server_id`(`management_server_id`) REFERENCES `mshost`(`id`) ON DELETE CASCADE, + KEY `i_management_server_details__name__value` (`name`(128),`value`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + -- Add checkpoint tracking fields to backups table for incremental backup support CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for the next incremental backup"'); diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index 2dd28e8e628f..75bcaf9a7ddf 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -48,6 +48,12 @@ cloud-api ${project.version} + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + compile + diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java new file mode 100644 index 000000000000..fcaa2a22e341 --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "management_server_details") +public class ManagementServerHostDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "management_server_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + @Column(name = "display") + private boolean display = true; + + public ManagementServerHostDetailVO(long poolId, String name, String value, boolean display) { + this.resourceId = poolId; + this.name = name; + this.value = value; + this.display = display; + } + + public ManagementServerHostDetailVO() { + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java new file mode 100644 index 000000000000..f3ede42bbe4f --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.cluster.ManagementServerHostDetailVO; +import com.cloud.utils.db.GenericDao; + +public interface ManagementServerHostDetailsDao extends GenericDao, ResourceDetailsDao { +} + diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java new file mode 100644 index 000000000000..5865bee0926b --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +import com.cloud.cluster.ManagementServerHostDetailVO; + +public class ManagementServerHostDetailsDaoImpl extends ResourceDetailsDaoBase implements ManagementServerHostDetailsDao, ScopedConfigStorage { + + public ManagementServerHostDetailsDaoImpl() { + } + + @Override + public ConfigKey.Scope getScope() { + return ConfigKey.Scope.ManagementServer; + } + + @Override + public String getConfigValue(long id, String key) { + ManagementServerHostDetailVO vo = findDetail(id, key); + return vo == null ? null : vo.getValue(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new ManagementServerHostDetailVO(resourceId, key, value, display)); + } +} diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 6da5dda967d0..97a1a42b5595 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -160,6 +160,9 @@ import com.cloud.api.query.vo.NetworkOfferingJoinVO; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.cluster.dao.ManagementServerHostDetailsDao; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.AccountVlanMapVO; import com.cloud.dc.ClusterDetailsDao; @@ -469,6 +472,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati @Inject ImageStoreDetailsDao _imageStoreDetailsDao; @Inject + ManagementServerHostDao managementServerHostDao; + @Inject + ManagementServerHostDetailsDao managementServerHostDetailsDao; + @Inject MessageBus messageBus; @Inject AgentManager _agentManager; @@ -885,6 +892,13 @@ public String updateConfiguration(final long userId, final String name, final St } break; + case ManagementServer: + final ManagementServerHostVO managementServer = managementServerHostDao.findById(resourceId); + Preconditions.checkState(managementServer != null); + resourceType = ApiCommandResourceType.ManagementServer; + managementServerHostDetailsDao.addDetail(resourceId, name, value, true); + break; + default: throw new InvalidParameterValueException("Scope provided is invalid"); } @@ -1032,8 +1046,9 @@ public Configuration updateConfiguration(final UpdateCfgCmd cmd) throws InvalidP String value = cmd.getValue(); final Long zoneId = cmd.getZoneId(); final Long clusterId = cmd.getClusterId(); - final Long storagepoolId = cmd.getStoragepoolId(); + final Long storagepoolId = cmd.getStoragePoolId(); final Long imageStoreId = cmd.getImageStoreId(); + final Long managementServerId = cmd.getManagementServerId(); Long accountId = cmd.getAccountId(); Long domainId = cmd.getDomainId(); // check if config value exists @@ -1113,6 +1128,11 @@ public Configuration updateConfiguration(final UpdateCfgCmd cmd) throws InvalidP id = imageStoreId; paramCountCheck++; } + if (managementServerId != null) { + scope = ConfigKey.Scope.ManagementServer.toString(); + id = managementServerId; + paramCountCheck++; + } if (paramCountCheck > 1) { throw new InvalidParameterValueException("cannot handle multiple IDs, provide only one ID corresponding to the scope"); @@ -1168,6 +1188,7 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr final Long accountId = cmd.getAccountId(); final Long domainId = cmd.getDomainId(); final Long imageStoreId = cmd.getImageStoreId(); + final Long managementServerId = cmd.getManagementServerId(); ConfigKey configKey = null; Optional optionalValue; String defaultValue; @@ -1201,6 +1222,7 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr scopeMap.put(ConfigKey.Scope.Account.toString(), accountId); scopeMap.put(ConfigKey.Scope.StoragePool.toString(), storagepoolId); scopeMap.put(ConfigKey.Scope.ImageStore.toString(), imageStoreId); + scopeMap.put(ConfigKey.Scope.ManagementServer.toString(), managementServerId); ParamCountPair paramCountPair = getParamCount(scopeMap); id = paramCountPair.getId(); @@ -1297,6 +1319,16 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr newValue = optionalValue.isPresent() ? optionalValue.get().toString() : defaultValue; break; + case ManagementServer: + final ManagementServerHostVO managementServer = managementServerHostDao.findById(id); + if (managementServer == null) { + throw new InvalidParameterValueException("unable to find management server by id " + id); + } + managementServerHostDetailsDao.removeDetail(id, name); + optionalValue = Optional.ofNullable(configKey != null ? configKey.valueIn(id) : config.getValue()); + newValue = optionalValue.isPresent() ? optionalValue.get().toString() : defaultValue; + break; + default: if (!_configDao.update(name, category, defaultValue)) { logger.error("Failed to reset configuration option, name: {}, defaultValue: {}", name, defaultValue); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index d4266496e982..f0b257670ef4 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -2362,6 +2362,7 @@ public Pair, Integer> searchForConfigurations(fina final Long clusterId = cmd.getClusterId(); final Long storagepoolId = cmd.getStoragepoolId(); final Long imageStoreId = cmd.getImageStoreId(); + final Long managementServerId = cmd.getManagementServerId(); Long accountId = cmd.getAccountId(); Long domainId = cmd.getDomainId(); final String groupName = cmd.getGroupName(); @@ -2415,6 +2416,11 @@ public Pair, Integer> searchForConfigurations(fina id = imageStoreId; paramCountCheck++; } + if (managementServerId != null) { + scope = ConfigKey.Scope.ManagementServer; + id = managementServerId; + paramCountCheck++; + } if (paramCountCheck > 1) { throw new InvalidParameterValueException("cannot handle multiple IDs, provide only one ID corresponding to the scope"); diff --git a/ui/src/components/view/SettingsTab.vue b/ui/src/components/view/SettingsTab.vue index 0e0ee33f2800..476c20b5c068 100644 --- a/ui/src/components/view/SettingsTab.vue +++ b/ui/src/components/view/SettingsTab.vue @@ -87,6 +87,7 @@ export default { } }, created () { + console.log('---------------', this.$route.meta.name) switch (this.$route.meta.name) { case 'account': this.scopeKey = 'accountid' @@ -106,6 +107,9 @@ export default { case 'imagestore': this.scopeKey = 'imagestoreuuid' break + case 'managementserver': + this.scopeKey = 'managementserverid' + break default: this.scopeKey = '' } diff --git a/ui/src/config/section/infra/managementServers.js b/ui/src/config/section/infra/managementServers.js index d2d11d5b25d0..4bf54943fd85 100644 --- a/ui/src/config/section/infra/managementServers.js +++ b/ui/src/config/section/infra/managementServers.js @@ -39,6 +39,10 @@ export default { name: 'details', component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) }, + { + name: 'settings', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/SettingsTab.vue'))) + }, { name: 'management.server.peers', component: shallowRef(defineAsyncComponent(() => import('@/views/infra/ManagementServerPeerTab.vue'))) From 82d062e3f5196ebe306f59b596e2827d54812bd8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Apr 2026 13:37:57 +0530 Subject: [PATCH 125/129] fix checkstyle error Signed-off-by: Abhishek Kumar --- .../cloudstack/api/command/admin/config/ListCfgsByCmd.java | 1 - .../cloudstack/api/command/admin/config/ResetCfgCmd.java | 7 +++---- .../cloudstack/api/command/admin/config/UpdateCfgCmd.java | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index a7757cf0ee38..055443934609 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -20,7 +20,6 @@ import java.util.List; import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java index 5e7d38c830f7..3c3c36b29d7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java @@ -23,17 +23,16 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.api.response.ManagementServerResponse; -import org.apache.cloudstack.framework.config.ConfigKey; - import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.user.Account; import com.cloud.utils.Pair; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index c6fb62b4ff8a..028d5c962a71 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -16,9 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.admin.config; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; @@ -29,6 +27,7 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; @@ -37,6 +36,7 @@ import org.apache.commons.lang3.StringUtils; import com.cloud.user.Account; +import com.cloud.utils.crypt.DBEncryptionUtil; @APICommand(name = "updateConfiguration", description = "Updates a configuration.", responseObject = ConfigurationResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) From fac62adfe31ec697202966cee07bd9bf5870fa40 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Apr 2026 13:44:48 +0530 Subject: [PATCH 126/129] build fix Signed-off-by: Abhishek Kumar --- .../java/com/cloud/configuration/ConfigurationManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 97a1a42b5595..ad31cfbaef9a 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1129,7 +1129,7 @@ public Configuration updateConfiguration(final UpdateCfgCmd cmd) throws InvalidP paramCountCheck++; } if (managementServerId != null) { - scope = ConfigKey.Scope.ManagementServer.toString(); + scope = ConfigKey.Scope.ManagementServer; id = managementServerId; paramCountCheck++; } From 830044d88f61940aa2568ca6ce837c564720b931 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Apr 2026 13:45:14 +0530 Subject: [PATCH 127/129] make bind address managementserver scoped Signed-off-by: Abhishek Kumar --- .../cloudstack/veeam/VeeamControlServer.java | 21 ++++++++++++------- .../cloudstack/veeam/VeeamControlService.java | 4 +++- .../veeam/VeeamControlServiceImpl.java | 15 +++++++++++++ .../cloudstack/veeam/api/ApiRouteHandler.java | 9 ++++---- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java index a70babe9b279..48a27802dd38 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java @@ -77,10 +77,17 @@ public void startIfEnabled() throws Exception { StringUtils.isNotEmpty(keystorePassword) && StringUtils.isNotEmpty(keyManagerPassword) && Files.exists(Paths.get(keystorePath)); - final String bind = VeeamControlService.BindAddress.value(); + long managementServerHostId = veeamControlService.getCurrentManagementServerHostId(); + final String bindAddress = VeeamControlService.BindAddress.valueIn(managementServerHostId); + final String bindHost = StringUtils.trimToNull(bindAddress); final int port = VeeamControlService.Port.value(); + final String bindDisplay = bindHost == null ? + String.format("all interfaces, port: %d", port) : + String.format("host: %s, port: %d", bindHost, port); String ctxPath = VeeamControlService.ContextPath.value(); - LOGGER.info("Veeam Control server - bind: {}, port: {}, context: {} with {} handlers", bind, port, ctxPath, + LOGGER.info("Veeam Control server - {}, context: {} with {} handlers", + bindDisplay, + ctxPath, routeHandlers != null ? routeHandlers.size() : 0); @@ -102,20 +109,20 @@ public void startIfEnabled() throws Exception { new SslConnectionFactory(sslContextFactory, "http/1.1"), new HttpConnectionFactory(https) ); - httpsConnector.setHost(bind); + httpsConnector.setHost(bindHost); httpsConnector.setPort(port); server.addConnector(httpsConnector); - LOGGER.info("Veeam Control API server HTTPS enabled on {}:{}", bind, port); + LOGGER.info("Veeam Control API server HTTPS enabled on {}", bindDisplay); } else { final HttpConfiguration http = new HttpConfiguration(); final ServerConnector httpConnector = new ServerConnector(server, new HttpConnectionFactory(http)); - httpConnector.setHost(bind); + httpConnector.setHost(bindHost); httpConnector.setPort(port); server.addConnector(httpConnector); LOGGER.warn("Veeam Control API server HTTPS is NOT configured (missing keystore path/passwords). " + - "Starting HTTP on {}:{} instead.", bind, port); + "Starting HTTP on {} instead.", bindDisplay); } final ServletContextHandler ctx = @@ -140,7 +147,7 @@ public void startIfEnabled() throws Exception { server.start(); - LOGGER.info("Started Veeam Control API server on {}:{} with context {}", bind, port, ctxPath); + LOGGER.info("Started Veeam Control API server on {}:{} with context {}", bindDisplay, port, ctxPath); } @NotNull diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java index 159d7eead066..ae7c00ad94db 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java @@ -31,7 +31,8 @@ public interface VeeamControlService extends PluggableService, Configurable { ConfigKey Enabled = new ConfigKey<>("Advanced", Boolean.class, "integration.veeam.control.enabled", "false", "Enable the Veeam Integration REST API server", false); ConfigKey BindAddress = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.bind.address", - "127.0.0.1", "Bind address for Veeam Integration REST API server", false); + "", "Bind address for Veeam Integration REST API server", false, + ConfigKey.Scope.ManagementServer); ConfigKey Port = new ConfigKey<>("Advanced", Integer.class, "integration.veeam.control.port", "8090", "Port for Veeam Integration REST API server", false); ConfigKey ContextPath = new ConfigKey<>("Advanced", String.class, "integration.veeam.control.context.path", @@ -56,6 +57,7 @@ public interface VeeamControlService extends PluggableService, Configurable { "", "Comma-separated list of CIDR blocks representing clients allowed to access the API. " + "If empty, all clients will be allowed. Example: '192.168.1.1/24,192.168.2.100/32", true); + long getCurrentManagementServerHostId(); List getAllowedClientCidrs(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java index a00d6bd5b836..b52a113648b4 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java @@ -21,16 +21,24 @@ import java.util.List; import java.util.stream.Collectors; +import javax.inject.Inject; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.utils.cache.SingleCache; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.veeam.utils.DataUtil; import org.apache.commons.lang3.StringUtils; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.net.NetUtils; public class VeeamControlServiceImpl extends ManagerBase implements VeeamControlService { + @Inject + ManagementServerHostDao managementServerHostDao; + private List routeHandlers; private VeeamControlServer veeamControlServer; private SingleCache> allowedClientCidrsCache; @@ -63,6 +71,13 @@ public void setRouteHandlers(final List routeHandlers) { this.routeHandlers = routeHandlers; } + @Override + public long getCurrentManagementServerHostId() { + ManagementServerHostVO hostVO = + managementServerHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + return hostVO.getId(); + } + @Override public List getAllowedClientCidrs() { return allowedClientCidrsCache.get(); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java index be71164d672b..f8b82d7f25fb 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.veeam.api; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -40,7 +39,7 @@ import org.apache.cloudstack.veeam.api.dto.Version; import org.apache.cloudstack.veeam.utils.Negotiation; -import com.cloud.utils.UuidUtils; +import com.cloud.user.AccountService; import com.cloud.utils.component.ManagerBase; public class ApiRouteHandler extends ManagerBase implements RouteHandler { @@ -49,6 +48,9 @@ public class ApiRouteHandler extends ManagerBase implements RouteHandler { @Inject ServerAdapter serverAdapter; + @Inject + AccountService accountService; + @Override public boolean canHandle(String method, String path) { return getSanitizedPath(path).startsWith("/api"); @@ -97,8 +99,7 @@ protected Api createApiObject(String basePath) { /* ---------------- Product info ---------------- */ ProductInfo productInfo = new ProductInfo(); - productInfo.setInstanceId(UuidUtils.nameUUIDFromBytes( - VeeamControlService.BindAddress.value().getBytes(StandardCharsets.UTF_8)).toString()); + productInfo.setInstanceId(accountService.getSystemAccount().getUuid()); productInfo.name = VeeamControlService.PLUGIN_NAME; productInfo.version = Version.fromPackageAndCSVersion(true); From 0c83842fabebc5694f11a09b19a6337814469206 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Apr 2026 15:38:05 +0530 Subject: [PATCH 128/129] fix updateconfiguration response scope Signed-off-by: Abhishek Kumar --- .../cloudstack/api/command/admin/config/UpdateCfgCmd.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index 028d5c962a71..9db9529dc8d8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.lang3.StringUtils; import com.cloud.user.Account; @@ -203,6 +204,9 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getDomainId() != null) { response.setScope("domain"); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name().toLowerCase()); + } return response; } } From c9a55c866e1574f7869b1ae9a9f66ee3e474ee53 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Apr 2026 15:38:25 +0530 Subject: [PATCH 129/129] fix image_transfer column name Signed-off-by: Abhishek Kumar --- .../src/main/resources/META-INF/db/schema-42210to42300.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 9e928fe0c77c..ce66f0d26dd0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', `data_center_id` bigint unsigned NOT NULL COMMENT 'Data Center ID', `backup_id` bigint unsigned COMMENT 'Backup ID', - `disk_id` bigint unsigned NOT NULL COMMENT 'Disk/Volume ID', + `volume_id` bigint unsigned NOT NULL COMMENT 'Volume ID', `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', `file` varchar(255) COMMENT 'File for the file backend',