Introduce new backup provider (KBOSS) (#12758)

Co-authored-by: João Jandre <joao@scclouds.com.br>
Co-authored-by: Bernardo De Marco Gonçalves <bernardomg2004@gmail.com>
Co-authored-by: Fabricio Duarte <fabricio.duarte.jr@gmail.com>
Co-authored-by: GaOrtiga <49285692+GaOrtiga@users.noreply.github.com>

Author: 5 people

agent/conf/agent.properties

@@ -488,3 +488,15 @@ iscsi.session.cleanup.enabled=false
 # Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as
 # -io vddk-thumbprint=<value>. If unset, CloudStack computes it on the KVM host via openssl.
 #vddk.thumbprint=
+
+# Timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KBOSS plugin.
+# If a value of 0 or less is informed, the default will be used.
+# qcow2.delta.merge.timeout=259200
+
+# Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+# the same time.
+# backup.validation.max.concurrent.operations.per.host=
+
+# Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+# done at the same time.
+# backup.compression.max.concurrent.operations.per.host=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -170,7 +170,8 @@ public class AgentProperties{
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
     /**
-     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * The timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KBOSS plugin.
+     * If a value of 0 or less is informed, the default will be used.<br>
      * This configuration is only considered if libvirt.events.enabled is also true. <br>
      * Data type: Integer.<br>
      * Default value: <code>259200</code>
@@ -953,6 +954,18 @@ public Property<Integer> getWorkers() {
     public static final Property<Boolean> VM_NETWORK_MACIP_STATIC = new Property<>("vm.network.macip.static", false, Boolean.class);
 
 
+    /**
+     * Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+     * the same time.
+     */
+    public static final Property<Integer> BACKUP_VALIDATION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.validation.max.concurrent.operations.per.host", null, Integer.class);
+
+    /**
+     * Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+     * done at the same time.
+     */
+    public static final Property<Integer> BACKUP_COMPRESSION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.compression.max.concurrent.operations.per.host", null, Integer.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/com/cloud/agent/api/to/DataObjectType.java

@@ -19,5 +19,5 @@
 package com.cloud.agent.api.to;
 
 public enum DataObjectType {
-    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE
+    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE, BACKUP
 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -668,6 +668,7 @@ public class EventTypes {
     public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC";
     public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT";
     public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP";
+    public static final String EVENT_SCREENSHOT_DOWNLOAD = "BACKUP.VALIDATION.SCREENSHOT.DOWNLOAD";
 
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 
 import org.apache.cloudstack.backup.Backup;
+import org.apache.cloudstack.backup.BackupProvider;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
@@ -94,10 +95,10 @@ public interface HypervisorGuru extends Adapter {
     Map<String, String> getClusterSettings(long vmId);
 
     VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, long accountId, long userId,
-                                                  String vmInternalName, Backup backup) throws Exception;
+                                                  String vmInternalName, Backup backup, BackupProvider backupProvider) throws Exception;
 
     boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo,
-                                                 VirtualMachine vm, long poolId, Backup backup) throws Exception;
+                                                 VirtualMachine vm, long poolId, Backup backup, BackupProvider backupProvider) throws Exception;
     /**
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *

api/src/main/java/com/cloud/storage/Storage.java

@@ -35,7 +35,8 @@ public static enum ImageFormat {
         VDI(true, true, false, "vdi"),
         TAR(false, false, false, "tar"),
         ZIP(false, false, false, "zip"),
-        DIR(false, false, false, "dir");
+        DIR(false, false, false, "dir"),
+        PNG(false, false, false, "png");
 
         private final boolean supportThinProvisioning;
         private final boolean supportSparse;

api/src/main/java/com/cloud/storage/Volume.java

@@ -60,7 +60,9 @@ enum State {
         UploadError(false, "Volume upload encountered some error"),
         UploadAbandoned(false, "Volume upload is abandoned since the upload was never initiated within a specified time"),
         Attaching(true, "The volume is attaching to a VM from Ready state."),
-        Restoring(true, "The volume is being restored from backup.");
+        Restoring(true, "The volume is being restored from backup."),
+        Consolidating(true, "The volume is being flattened."),
+        RestoreError(false, "The volume restore encountered an error.");
 
         boolean _transitional;
 
@@ -153,6 +155,10 @@ public String getDescription() {
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Destroy, Event.RestoreRequested, Restoring, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreSucceeded, Ready, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreFailed, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Ready, Event.ConsolidationRequested, Consolidating, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationSucceeded, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationFailed, RestoreError, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(RestoreError, Event.RestoreFailed, RestoreError, null));
         }
     }
 
@@ -179,7 +185,8 @@ enum Event {
         OperationTimeout,
         RestoreRequested,
         RestoreSucceeded,
-        RestoreFailed;
+        RestoreFailed,
+        ConsolidationRequested
     }
 
     /**

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -114,7 +114,7 @@ Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Lo
 
     Volume attachVolumeToVM(AttachVolumeCmd command);
 
-    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS);
+    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS, boolean allowAttachOnRestoring);
 
     Volume detachVolumeViaDestroyVM(long vmId, long volumeId);
 
@@ -189,7 +189,7 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
 
     boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool);
 
-    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
+    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge, Boolean countDisplayFalseInResourceCount);
 
     void destroyVolume(long volumeId);
 

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -254,14 +254,14 @@ public interface ResourceLimitService {
     void updateTaggedResourceLimitsAndCountsForAccounts(List<AccountResponse> responses, String tag);
     void updateTaggedResourceLimitsAndCountsForDomains(List<DomainResponse> responses, String tag);
     void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List<Reserver> reservations) throws ResourceAllocationException;
-    List<String> getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering);
+    List<String> getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering, Boolean enforceResourceLimitOnDisplayFalse);
     void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean display, Long currentSize, Long newSize,
             DiskOffering currentOffering, DiskOffering newOffering, List<Reserver> reservations) throws ResourceAllocationException;
 
     void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List<Reserver> reservations) throws ResourceAllocationException;
 
     void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
-    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering, Boolean countDisplayFalseInResourceCount);
 
     void updateVmResourceCountForTemplateChange(long accountId, Boolean display, ServiceOffering offering, VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate);
 
@@ -276,8 +276,8 @@ void updateVolumeResourceCountForDiskOfferingChange(long accountId, Boolean disp
     void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Reserver> reservations) throws ResourceAllocationException;
-    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
-    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceLimit);
+    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceCount);
 
     void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu,
             Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template, List<Reserver> reservations) throws ResourceAllocationException;

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -75,10 +75,11 @@ public interface UserVmService {
      * Destroys one virtual machine
      *
      * @param cmd the API Command Object containg the parameters to use for this service action
+     * @param checkExpunge
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
-    UserVm destroyVm(DestroyVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException;
+    UserVm destroyVm(DestroyVMCmd cmd, boolean checkExpunge) throws ResourceUnavailableException, ConcurrentOperationException;
 
     /**
      * Destroys one virtual machine