MDEV-39795: Assertion `n_reserved > 0' failed

mysql-test/suite/encryption/r/innodb_encrypt_compression_algorithm.result

@@ -0,0 +1,34 @@
+#
+# MDEV 39795 : Assertion `n_reserved > 0' failed
+#
+SET GLOBAL innodb_encrypt_tables = ON;
+SET GLOBAL innodb_compression_algorithm = 'none';
+CREATE TABLE t1 (
+id INT AUTO_INCREMENT PRIMARY KEY,
+col1 INT,
+col2 INT,
+col3 INT
+) ENGINE=InnoDB ROW_FORMAT = Compact PAGE_COMPRESSED=1;
+INSERT INTO t1 (col1, col2, col3) VALUES (1, 10, 100);
+FLUSH TABLES t1 FOR EXPORT;
+# restart
+ALTER TABLE t1 ADD INDEX idx_col1 (col1);
+FLUSH TABLES t1 FOR EXPORT;
+# restart
+ALTER TABLE t1 ADD INDEX idx_col2_col3 (col2, col3);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `col1` int(11) DEFAULT NULL,
+  `col2` int(11) DEFAULT NULL,
+  `col3` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `idx_col1` (`col1`),
+  KEY `idx_col2_col3` (`col2`,`col3`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci ROW_FORMAT=COMPACT `PAGE_COMPRESSED`=1
+SELECT * FROM t1;
+id	col1	col2	col3
+1	1	10	100
+# Cleanup
+DROP TABLE t1;

mysql-test/suite/encryption/t/innodb_encrypt_compression_algorithm.test

@@ -0,0 +1,29 @@
+-- source include/have_innodb.inc
+-- source include/have_file_key_management_plugin.inc
+
+--echo #
+--echo # MDEV 39795 : Assertion `n_reserved > 0' failed
+--echo #
+
+SET GLOBAL innodb_encrypt_tables = ON;
+SET GLOBAL innodb_compression_algorithm = 'none';
+CREATE TABLE t1 (
+  id INT AUTO_INCREMENT PRIMARY KEY,
+  col1 INT,
+  col2 INT,
+  col3 INT
+) ENGINE=InnoDB ROW_FORMAT = Compact PAGE_COMPRESSED=1;
+INSERT INTO t1 (col1, col2, col3) VALUES (1, 10, 100);
+FLUSH TABLES t1 FOR EXPORT;
+
+--source include/restart_mysqld.inc
+ALTER TABLE t1 ADD INDEX idx_col1 (col1);
+FLUSH TABLES t1 FOR EXPORT;
+
+--source include/restart_mysqld.inc
+ALTER TABLE t1 ADD INDEX idx_col2_col3 (col2, col3);
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+--echo # Cleanup
+DROP TABLE t1;

storage/innobase/buf/buf0flu.cc

@@ -658,17 +658,18 @@ static byte *buf_page_encrypt(fil_space_t *space, buf_page_t *bpage, byte *s,
 
   const bool full_crc32= space->full_crc32();
 
-  if (!encrypted && !page_compressed)
+  if (!encrypted)
   {
-    /* No need to encrypt or compress. Clear key-version & crypt-checksum. */
+    /* No need to encrypt. Clear key-version & crypt-checksum. */
     static_assert(FIL_PAGE_FCRC32_KEY_VERSION % 4 == 0, "alignment");
     static_assert(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION % 4 == 2,
                   "not perfect alignment");
     if (full_crc32)
       memset_aligned<4>(s + FIL_PAGE_FCRC32_KEY_VERSION, 0, 4);
     else
       memset_aligned<2>(s + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8);
-    return s;
+    if (!page_compressed)
+      return s;
   }
 
   static_assert(FIL_PAGE_FCRC32_END_LSN % 4 == 0, "alignment");

storage/innobase/fsp/fsp0fsp.cc

@@ -1684,6 +1684,7 @@ fseg_create(fil_space_t *space, ulint byte_offset, mtr_t *mtr, dberr_t *err,
 	fseg_inode_t*	inode;
 	ib_id_t		seg_id;
 	uint32_t	n_reserved = 0;
+	bool		reserved = false;
 
 	DBUG_ENTER("fseg_create");
 
@@ -1707,20 +1708,24 @@ fseg_create(fil_space_t *space, ulint byte_offset, mtr_t *mtr, dberr_t *err,
 
 inode_alloc:
 	inode = fsp_alloc_seg_inode(space, header, &iblock, mtr, err);
+	if (*err == DB_DECRYPTION_FAILED)
+		DBUG_RETURN(nullptr);
 
 	if (!inode) {
 		block = nullptr;
 reserve_extent:
-		if (!has_done_reservation && !n_reserved) {
+		if (!has_done_reservation && !reserved) {
 			*err = fsp_reserve_free_extents(&n_reserved, space, 2,
 							FSP_NORMAL, mtr);
 			if (UNIV_UNLIKELY(*err != DB_SUCCESS)) {
 				DBUG_RETURN(nullptr);
 			}
 
-			ut_ad(n_reserved > 0);
-			/* Extents reserved successfully. So
-			try allocating the page or inode */
+			/* Extents (or pages for small table)
+			reserved successfully.
+			See fsp_reserve_free_extents()
+			So try allocating the page or inode */
+			reserved = true;
 			if (inode) {
 				goto page_alloc;
 			}

storage/innobase/handler/handler0alter.cc

@@ -918,6 +918,10 @@ my_error_innodb(
 	case DB_CORRUPTION:
 		my_error(ER_NOT_KEYFILE, MYF(0), table);
 		break;
+	case DB_DECRYPTION_FAILED:
+		my_error(ER_GET_ERRMSG, MYF(0), error,
+		         ut_strerr(error), "InnoDB");
+		break;
 	case DB_TOO_BIG_RECORD: {
 		/* Note that in page0zip.ic page_zip_rec_needs_ext() rec_size
 		is limited to COMPRESSED_REC_MAX_DATA_SIZE (16K) or