feat: pgbackrest test

tests/robot/Lib/lib.robot

@@ -103,7 +103,7 @@ Insert Test Record
     ${res}=  Execute Query   ${MASTERHOST}  select * from test_insert_robot where id=${RID}   dbname=${database}
     Should Be True   """${EXPECTED}""" in """${res}"""   msg=[insert test record] Expected string ${EXPECTED} not found on ${MASTERHOST} : res: ${res}
     Log To Console  Test records found on ${MASTERHOST}
-    [Return]   ${RID}   ${EXPECTED}
+    RETURN   ${RID}   ${EXPECTED}
 
 Check Test Record
     [Arguments]    ${pod_name}   ${RID}   ${EXPECTED}    ${database}=postgres

tests/robot/Lib/pgsLibrary.py

@@ -781,6 +781,145 @@ def check_last_backup_id(self):
         last_backup_id = health_json["storage"]["lastSuccessful"]["id"]
         return last_backup_id
 
+    def pgbackrest_backup_exists(self, backup_id):
+        try:
+            response = requests.get(
+                f"{self._scheme}://postgres-backup-daemon:8080/backup/status/{backup_id}",
+                verify=False,
+                timeout=10
+            )
+            if response.status_code == 404:
+                logging.info("PgBackRest backup %s was not found through backup daemon /backup/status", backup_id)
+                return False
+            response.raise_for_status()
+            status = response.text.strip()
+            if status == "Backup Done":
+                logging.info("PgBackRest backup %s found through backup daemon /backup/status", backup_id)
+                return True
+            logging.info("PgBackRest backup %s is not ready yet. Backup daemon status: %s", backup_id, status)
+        except Exception as e:
+            logging.info("Cannot check backup daemon status for PgBackRest backup %s: %s", backup_id, e)
+
+        status = self.get_pgbackrest_sidecar_backup_status(backup_id)
+        if self._pgbackrest_backup_matches(backup_id, status):
+            logging.info("PgBackRest backup %s found through backrest /status", backup_id)
+            return True
+
+        backups = self.get_pgbackrest_sidecar_backup_list()
+        if self._pgbackrest_backup_found_in_list(backup_id, backups):
+            logging.info("PgBackRest backup %s found through backrest /list", backup_id)
+            return True
+
+        return False
+
+    def get_pgbackrest_backup_list(self):
+        response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10)
+        response.raise_for_status()
+        backups = response.json()
+        logging.info("Backup daemon backup list: {}".format(backups))
+        return backups
+
+    def get_pgbackrest_sidecar_backup_status(self, backup_id):
+        for service in ["backrest", "backrest-headless"]:
+            try:
+                response = requests.get(
+                    "http://{}:3000/status?timestamp={}".format(service, backup_id),
+                    timeout=10
+                )
+                response.raise_for_status()
+                status = response.json()
+                logging.info("PgBackRest status from %s for %s: %s", service, backup_id, status)
+                if self._pgbackrest_backup_matches(backup_id, status):
+                    return status
+            except Exception as e:
+                logging.info("Cannot get PgBackRest status from %s for %s: %s", service, backup_id, e)
+        return {}
+
+    def get_pgbackrest_sidecar_backup_list(self):
+        errors = {}
+        for service in ["backrest", "backrest-headless"]:
+            try:
+                response = requests.get("http://{}:3000/list".format(service), timeout=10)
+                response.raise_for_status()
+                backups = response.json()
+                logging.info("PgBackRest list from %s: %s", service, backups)
+                return backups
+            except Exception as e:
+                errors[service] = str(e)
+                logging.info("Cannot get PgBackRest list from %s: %s", service, e)
+        logging.info("Cannot get PgBackRest list from sidecar services: %s", errors)
+        return []
+
+    def _pgbackrest_backup_matches(self, backup_id, backup):
+        if not isinstance(backup, dict):
+            return False
+        annotation = backup.get("annotation") or {}
+        return annotation.get("timestamp") == backup_id and not backup.get("error", False)
+
+    def _pgbackrest_backup_found_in_list(self, backup_id, backups):
+        if isinstance(backups, dict):
+            if backup_id in backups:
+                return True
+            backups = backups.values()
+        if not isinstance(backups, list):
+            return False
+        for backup in backups:
+            if self._pgbackrest_backup_matches(backup_id, backup):
+                return True
+        return False
+
+    def restore_pgbackrest_backup(self, backup_id):
+        pod = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        command = "cd /maintenance/recovery && SET={} python3 pg_back_rest_recovery.py".format(backup_id)
+        logging.info("Start PgBackRest restore from backup daemon pod {} with backup id {}".format(
+            pod.metadata.name, backup_id))
+        output, errors = self.execute_in_pod(pod.metadata.name, command)
+        logging.info("PgBackRest restore output: {}".format(output))
+        if errors:
+            logging.info("PgBackRest restore stderr: {}".format(errors))
+        return output
+
+    def get_backup_daemon_restart_count(self):
+        pod = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        restart_count = 0
+        for container_status in pod.status.container_statuses or []:
+            restart_count += container_status.restart_count
+        logging.info("Backup daemon pod {} restart count: {}".format(pod.metadata.name, restart_count))
+        return restart_count
+
+    def get_pgbackrest_prerequisite_status(self):
+        status = {
+            "storage_type": None,
+            "backup_daemon_pod": None,
+            "pgbackrest_configmap_exists": False,
+            "pgbackrest_sidecar_pods": [],
+            "missing": []
+        }
+
+        backup_daemon = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        status["backup_daemon_pod"] = backup_daemon.metadata.name
+        status["storage_type"] = self.get_env_for_pod(backup_daemon, "STORAGE_TYPE")
+        if status["storage_type"] != "pgbackrest":
+            status["missing"].append("backup daemon STORAGE_TYPE is not pgbackrest")
+
+        try:
+            self.pl_lib.get_config_map("pgbackrest-conf", self._namespace)
+            status["pgbackrest_configmap_exists"] = True
+        except Exception:
+            status["missing"].append("pgbackrest-conf config map is absent")
+
+        pg_cluster_name = os.getenv("PG_CLUSTER_NAME", "patroni")
+        for pod in self.get_pods(label="pgcluster:{}".format(pg_cluster_name), status="Running"):
+            for container in pod.spec.containers:
+                if container.name == "pgbackrest-sidecar":
+                    status["pgbackrest_sidecar_pods"].append(pod.metadata.name)
+                    break
+        if not status["pgbackrest_sidecar_pods"]:
+            status["missing"].append("pgbackrest-sidecar container is absent in running patroni pods")
+
+        logging.info("PgBackRest prerequisite status: {}".format(status))
+        return status
+
     def schedule_evict(self, last_backup_id):
         health_json = requests.delete(f"{self._scheme}://postgres-backup-daemon:8081/evict?id={last_backup_id}", verify=False).json()
         return health_json

tests/robot/check_crud_user/check_crud_user.robot

@@ -20,7 +20,7 @@ Check User Creation
     Execute Query  pg-${PG_CLUSTER_NAME}  CREATE USER ${user_name};
     ${res}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT usename FROM pg_user;
     Should Be True   """${user_name}""" in """${res}"""   msg=[creating user] Expected user ${user_name} is not created in pg-${PG_CLUSTER_NAME}: res: ${res}
-    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name};
+    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER IF EXISTS ${user_name};
 
 Check User Creation With Password
     [Tags]  patroni simple  check_user
@@ -29,7 +29,7 @@ Check User Creation With Password
     Should Be True   """${user_name}""" in """${res}"""   msg=[creating user] Expected user ${user_name} is not created in pg-${PG_CLUSTER_NAME}: res: ${res}
     ${pass}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT passwd from pg_shadow where usename='${user_name}';
     Should Not Be Empty  ${pass}
-    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name};
+    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER IF EXISTS ${user_name};
 
 Check Update User
     [Tags]  patroni simple  check_user
@@ -42,13 +42,13 @@ Check Update User
     Should Be True   """${user_name}""" in """${res}"""   msg=[updating user] Expected user ${user_name} is not exist after alter in pg-${PG_CLUSTER_NAME}: res: ${res}
     ${current_pass}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT passwd from pg_shadow where usename='${user_name}';
     Should Not Be Equal  ${previous_pass}  ${current_pass}
-    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name};
+    [Teardown]  Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER IF EXISTS ${user_name};
 
 Check Deletion Of User
     [Tags]  patroni simple  check_user
     Execute Query  pg-${PG_CLUSTER_NAME}  CREATE USER ${user_name} WITH PASSWORD '${user_pass}';
     ${res}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT usename FROM pg_user;
     Should Be True   """${user_name}""" in """${res}"""   msg=[creating user] Expected user ${user_name} is not created in pg-${PG_CLUSTER_NAME}: res: ${res}
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name};
+    Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER IF EXISTS ${user_name};
     ${res}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT usename FROM pg_user;
     Should Not Be True   """${user_name}""" in """${res}"""   msg=[deleting user] User ${user_name} is not deleted from pg-${PG_CLUSTER_NAME}: res: ${res}

tests/robot/check_dbaas_adapter_api/check_dbaas_adapter_backup_api.robot

@@ -11,7 +11,7 @@ Backup Database By Dbaas Adapter
     Should Be Equal As Strings  ${resp.status_code}  202
     Dictionary Should Contain Key  ${resp.json()}  trackId
     ${trackId}=  Get From Dictionary  ${resp.json()}  trackId
-    [Return]  ${trackId}
+    RETURN  ${trackId}
 
 Check Backup Status By Dbaas Adapter
     [Arguments]  ${trackId}
@@ -26,7 +26,7 @@ Restore Database By Dbaas Adapter
     Should Be Equal As Strings  ${resp.status_code}  202
     Dictionary Should Contain Key  ${resp.json()}  trackId
     ${trackId}=  Get From Dictionary  ${resp.json()}  trackId
-    [Return]  ${trackId}
+    RETURN  ${trackId}
 
 Check Restore Status By Dbaas Adapter
     [Arguments]  ${trackId}

tests/robot/check_dbaas_adapter_api/keywords.robot

@@ -51,4 +51,4 @@ Check Database Creating By Dbaas Adapter
 Delete User And Database
     [Arguments]  ${db_name}  ${user_name}
     Delete Test DB  ${db_name}
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name}
+    Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER IF EXISTS ${user_name}

tests/robot/check_granular_api/check_granular_backups_list_auth_api.robot

@@ -39,7 +39,7 @@ Set Backups
     ${result}=  Get From Dictionary  ${resp.json()}  ${name_space}
     Set Test Variable  ${backups_in_namespace}  ${result}
     Log  ${result}
-    [Return]  ${result}
+    RETURN  ${result}
 
 *** Test Cases ***
 Check Backup Requests Status Endpoint

tests/robot/check_granular_api/check_granular_delete_backup_auth_api.robot

@@ -6,8 +6,8 @@ Resource          ../Lib/lib.robot
 
 *** Keywords ***
 Backup Not Exist
-    ${resp}=  Get Request  postgres_backup_daemon  /backup/status/${backup_id}?namespace=${name_space}
-    Should Be True  '${resp.status_code}'!=' ${200}'
+    ${resp}=  GET On Session  postgres_backup_daemon  /backup/status/${backup_id}?namespace=${name_space}  expected_status=any
+    Should Not Be Equal As Integers  ${resp.status_code}  200
 
 *** Test Cases ***
 Check Backup Requests Status Endpoint

tests/robot/check_granular_api/check_granular_restore_backup_auth_api.robot

@@ -57,7 +57,7 @@ Check Disabled Auth Regular Backup
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Database  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId  ${backup_id}
@@ -79,7 +79,7 @@ Check Disabled Auth Regular Backup
     ${resp}=  Get On Session  postgres_backup_daemon  url=/restore/status/${restore_id}
     Should Be Equal  ${resp.status_code}  ${200}
     #delete backup and drop database after test
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE IF EXISTS ${db_name}
+    Delete Database  ${db_name}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
 
@@ -110,7 +110,7 @@ Check Enabled Auth Regular Backup
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Database  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId  ${backup_id}
@@ -132,7 +132,7 @@ Check Enabled Auth Regular Backup
     ${res}=  Execute Query   pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}   dbname=${db_name}
     Should Be True   """${EXPECTED}""" in """${res}"""   msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup and drop database after test
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Database  ${db_name}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
 

tests/robot/check_granular_api/check_granular_restore_backup_with_new_names_auth.robot

@@ -74,9 +74,9 @@ Check Disabled Auth With Db Name Change
     END
     ${output}=  Execute Query   pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_catalog.pg_database WHERE datname = 'TaRgEt_TeSt_Bd'
     Should Be True  """TaRgEt_TeSt_Bd""" in """${output}"""
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE test_db
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE TaRgEt_TeSt_Bd
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE target_test_bd
+    Delete Database  test_db
+    Delete Database  TaRgEt_TeSt_Bd
+    Delete Database  target_test_bd
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
 
@@ -138,9 +138,9 @@ Check Enabled Auth With Db Name Change
     END
     ${output}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_catalog.pg_database WHERE datname = 'TaRgEt_TeSt_Bd'
     Should Be True  """TaRgEt_TeSt_Bd""" in """${output}"""
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE test_db
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE TaRgEt_TeSt_Bd
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP IF EXISTS DATABASE DATABASE target_test_bd
+    Delete Database  test_db
+    Delete Database  TaRgEt_TeSt_Bd
+    Delete Database  target_test_bd
     #delete backup after test
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}

tests/robot/check_granular_api/check_granular_restore_backup_with_roles_auth.robot

@@ -29,7 +29,7 @@ Check Disabled Auth With Roles
     Create New Role  ${db_role}
     Create Database With Owner  ${db_name}  ${db_role}
     ${RID}  ${EXPECTED}=  Insert Test Record  database=${db_name}
-    Execute Query  pg-${PG_CLUSTER_NAME}  ALTER TABLE test_gb_table OWNER TO ${db_name}  dbname=${db_name}
+    Execute Query  pg-${PG_CLUSTER_NAME}  ALTER TABLE test_insert_robot OWNER TO ${db_role}  dbname=${db_name}
     ${PGSSLMODE}=  Get Environment Variable  PGSSLMODE
     ${scheme}=  Set Variable If  '${PGSSLMODE}' == 'require'  https  http
     Create Session  postgres_backup_daemon  ${scheme}://postgres-backup-daemon:9000
@@ -47,7 +47,7 @@ Check Disabled Auth With Roles
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Database  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}   ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId=${backup_id}
@@ -68,8 +68,8 @@ Check Disabled Auth With Roles
     ${res}=  Execute Query  pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}   dbname=${db_name}
     Should Be True  """${EXPECTED}""" in """${res}"""   msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup and database after test
-    Execute Query  pg-${PG_CLUSTER_NAME}  drop database if exists ${db_name}
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP ROLE ${db_role}
+    Delete Database  ${db_name}
+    Execute Query  pg-${PG_CLUSTER_NAME}  DROP ROLE IF EXISTS ${db_role}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
 
@@ -88,7 +88,7 @@ Check Enabled Auth With Roles
     Create New Role  ${db_role}
     Create Database With Owner  ${db_name}  ${db_role}
     ${RID}  ${EXPECTED}=  Insert Test Record  database=${db_name}
-    Execute Query  pg-${PG_CLUSTER_NAME}  ALTER TABLE test_gb_table OWNER TO ${db_name}  dbname=${db_name}
+    Execute Query  pg-${PG_CLUSTER_NAME}  ALTER TABLE test_insert_robot OWNER TO ${db_role}  dbname=${db_name}
     Create Session  postgres_backup_daemon  ${scheme}://postgres-backup-daemon:9000  auth=${auth}
     ${name_space}=  Get Current Date  result_format=%Y%m%d%H%M
     ${array_db_name}=  Create List  ${db_name}
@@ -97,14 +97,14 @@ Check Enabled Auth With Roles
     &{headers}=  Create Dictionary  Content-Type=application/json
     ${resp}=  POST On Session  postgres_backup_daemon  /backup/request  data=${json_data}  headers=${headers}
     Should Be Equal  ${resp.status_code}  ${202}
-    ${restore_id}=  Get From Dictionary  ${resp.json()}  backupId
+    ${backup_id}=  Get From Dictionary  ${resp.json()}  backupId
     FOR  ${INDEX}  IN RANGE  60
-        ${resp}=  GET On Session  postgres_backup_daemon  url=/backup/status/${restore_id}?namespace=${name_space}
+        ${resp}=  GET On Session  postgres_backup_daemon  url=/backup/status/${backup_id}?namespace=${name_space}
         ${status}=  Get From Dictionary    ${resp.json()}    status
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Database  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId=${backup_id}
@@ -125,6 +125,7 @@ Check Enabled Auth With Roles
     ${res}=   Execute Query  pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}  dbname=${db_name}
     Should Be True  """${EXPECTED}""" in """${res}"""  msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup after test
-    Execute Query  pg-${PG_CLUSTER_NAME}  drop database if exists ${db_name}
+    Delete Database  ${db_name}
+    Execute Query  pg-${PG_CLUSTER_NAME}  DROP ROLE IF EXISTS ${db_role}
     ${resp}=  GET On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}