diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 64e9681a2..4a15ae897 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1609,11 +1609,11 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_STRING: // $size = $size * 4; // Convert utf8mb4 size to bytes - if ($size > 16777215) { + if ($size > Database::MAX_MEDIUMTEXT_BYTES) { return 'LONGTEXT'; } - if ($size > 65535) { + if ($size > Database::MAX_TEXT_BYTES) { return 'MEDIUMTEXT'; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 0f30d5ab4..e824ad58a 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -36,15 +36,6 @@ */ class SQLite extends MariaDB { - /** - * MariaDB byte ceilings for TEXT-family types, mirrored so PRAGMA-based - * introspection produces the same characterMaximumLength values that - * INFORMATION_SCHEMA.COLUMNS would on MariaDB. - */ - private const MARIADB_TEXT_BYTES = '65535'; - private const MARIADB_MEDIUMTEXT_BYTES = '16777215'; - private const MARIADB_LONGTEXT_BYTES = '4294967295'; - /** Suffix appended to every FTS5 virtual table name created by this adapter. */ private const FTS_TABLE_SUFFIX = '_fts'; @@ -3008,19 +2999,25 @@ private function parseSqliteColumnType(string $declaration): array break; } + /** + * MariaDB byte ceilings for TEXT-family types, mirrored so PRAGMA-based + * introspection produces the same characterMaximumLength values that + * INFORMATION_SCHEMA.COLUMNS would on MariaDB. + */ + if ($this->emulateMySQL) { switch ($dataType) { case 'text': - $result['characterMaximumLength'] = self::MARIADB_TEXT_BYTES; + $result['characterMaximumLength'] = '' . Database::MAX_TEXT_BYTES; break; case 'mediumtext': - $result['characterMaximumLength'] = self::MARIADB_MEDIUMTEXT_BYTES; + $result['characterMaximumLength'] = '' . Database::MAX_MEDIUMTEXT_BYTES; break; case 'longtext': case 'json': - $result['characterMaximumLength'] = self::MARIADB_LONGTEXT_BYTES; + $result['characterMaximumLength'] = '' . Database::MAX_LONGTEXT_BYTES; break; case 'tinyint': diff --git a/src/Database/Database.php b/src/Database/Database.php index 5a6cd97d7..d130734d9 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -115,6 +115,11 @@ class Database public const MAX_ARRAY_INDEX_LENGTH = 255; public const MAX_UID_DEFAULT_LENGTH = 36; + // Maximum byte capacity for TEXT + public const MAX_TEXT_BYTES = 65535; + public const MAX_MEDIUMTEXT_BYTES = 16777215; + public const MAX_LONGTEXT_BYTES = 4294967295; + // Min limits public const MIN_INT = -2147483648; diff --git a/src/Database/Validator/Attribute.php b/src/Database/Validator/Attribute.php index 143eaaf81..e3dc0044e 100644 --- a/src/Database/Validator/Attribute.php +++ b/src/Database/Validator/Attribute.php @@ -317,22 +317,22 @@ public function checkType(Document $attribute): bool break; case Database::VAR_TEXT: - if ($size > 65535) { - $this->message = 'Max size allowed for text is: 65535'; + if ($size > Database::MAX_TEXT_BYTES) { + $this->message = 'Max size allowed for text is: ' . Database::MAX_TEXT_BYTES; throw new DatabaseException($this->message); } break; case Database::VAR_MEDIUMTEXT: - if ($size > 16777215) { - $this->message = 'Max size allowed for mediumtext is: 16777215'; + if ($size > Database::MAX_MEDIUMTEXT_BYTES) { + $this->message = 'Max size allowed for mediumtext is: ' . Database::MAX_MEDIUMTEXT_BYTES; throw new DatabaseException($this->message); } break; case Database::VAR_LONGTEXT: - if ($size > 4294967295) { - $this->message = 'Max size allowed for longtext is: 4294967295'; + if ($size > Database::MAX_LONGTEXT_BYTES) { + $this->message = 'Max size allowed for longtext is: ' . Database::MAX_LONGTEXT_BYTES; throw new DatabaseException($this->message); } break; diff --git a/src/Database/Validator/ByteLength.php b/src/Database/Validator/ByteLength.php new file mode 100644 index 000000000..337dba74c --- /dev/null +++ b/src/Database/Validator/ByteLength.php @@ -0,0 +1,53 @@ +max = $max; + } + + public function getDescription(): string + { + return 'Value must be a valid string no longer than ' . $this->max . ' bytes'; + } + + public function isArray(): bool + { + return false; + } + + public function getType(): string + { + return self::TYPE_STRING; + } + + public function isValid(mixed $value): bool + { + if (!\is_string($value)) { + return false; + } + + if ($this->max !== 0 && \strlen($value) > $this->max) { + return false; + } + + return true; + } +} diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index a3050b963..4e3f66ec4 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -353,10 +353,22 @@ protected function checkForInvalidAttributeValues(Document $document, array $str $validators[] = new Sequence($this->idAttributeType, $attribute['$id'] === '$sequence'); break; - case Database::VAR_VARCHAR: case Database::VAR_TEXT: + $validators[] = new ByteLength($size); + $validators[] = new ByteLength(Database::MAX_TEXT_BYTES); + break; + case Database::VAR_MEDIUMTEXT: + $validators[] = new ByteLength($size); + $validators[] = new ByteLength(Database::MAX_MEDIUMTEXT_BYTES); + break; + case Database::VAR_LONGTEXT: + $validators[] = new ByteLength($size); + $validators[] = new ByteLength(Database::MAX_LONGTEXT_BYTES); + break; + + case Database::VAR_VARCHAR: case Database::VAR_STRING: $validators[] = new Text($size, min: 0); break; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 4f998372d..fc10463a6 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -971,6 +971,125 @@ public function testUpsertDocuments(): void } } + public function testTextByteTruncationCreate(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + // Byte-capacity validation relies on attribute metadata, which + // schemaless adapters don't store, so there is nothing to enforce. + if (!$database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $database->createCollection(__FUNCTION__); + + // A `text` attribute at its maximum allowed size. On MySQL/MariaDB this + // maps to a TEXT column, which is limited to 65,535 *bytes*. + $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false); + + // The Structure validator caps a TEXT column at its 65,535-byte capacity, + // measuring the value's actual byte length. A 20,000-char emoji value is + // 80,000 bytes (4 bytes per char in utf8mb4), so it exceeds the column's + // byte capacity and must be rejected up front with a clean + // StructureException, rather than letting the database raise error 1406 + // (data truncation). + $value = \str_repeat('📝', 20000); + $this->assertGreaterThan(65535, \strlen($value)); // exceeds the byte capacity + + $document = new Document([ + '$id' => 'first', + 'text' => $value, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + try { + $database->createDocument(__FUNCTION__, $document); + $this->fail('Expected StructureException for over-capacity text value'); + } catch (StructureException $e) { + $this->assertStringContainsString('65535 bytes', $e->getMessage()); + } + } + + public function testTextByteTruncationValid(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if (!$database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $database->createCollection(__FUNCTION__); + $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false); + + // A value that fills the column's full byte capacity is stored and + // round-trips intact. 65,535 ASCII chars are exactly 65,535 bytes, so + // byte-based validation accepts the whole column, where the previous + // char-based cap would have rejected anything over 16,383 chars. + $okValue = \str_repeat('a', 65535); + + $document = new Document([ + '$id' => 'first', + 'text' => $okValue, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $created = $database->createDocument(__FUNCTION__, $document); + $fetched = $database->getDocument(__FUNCTION__, $created->getId()); + $this->assertEquals($okValue, $fetched->getAttribute('text')); + } + + public function testTextByteTruncationUpdate(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if (!$database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $database->createCollection(__FUNCTION__); + $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false); + + $document = new Document([ + '$id' => 'first', + 'text' => \str_repeat('a', 16383), + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $created = $database->createDocument(__FUNCTION__, $document); + + // An oversized value is rejected on update, the same as on create. + $value = \str_repeat('📝', 20000); + $this->assertGreaterThan(65535, \strlen($value)); // exceeds the byte capacity + + try { + $database->updateDocument(__FUNCTION__, $created->getId(), $created->setAttribute('text', $value)); + $this->fail('Expected StructureException for over-capacity text value on update'); + } catch (StructureException $e) { + $this->assertStringContainsString('65535 bytes', $e->getMessage()); + } + } + public function testUpsertDocumentsInc(): void { /** @var Database $database */ diff --git a/tests/unit/Validator/StructureTest.php b/tests/unit/Validator/StructureTest.php index 10afeaa0a..71c90577c 100644 --- a/tests/unit/Validator/StructureTest.php +++ b/tests/unit/Validator/StructureTest.php @@ -368,6 +368,116 @@ public function testStringValidation(): void $this->assertEquals('Invalid document structure: Attribute "title" has invalid type. Value must be a valid string and no longer than 256 chars', $validator->getDescription()); } + public function testTextByteSafeValidationTooBig(): void + { + // A legacy `text` attribute whose declared size (1MB) exceeds the real + // 65,535-byte capacity of a TEXT column. Such attributes exist in older + // databases created before VAR_TEXT was capped, so the limit must come + // from the column type, not from the (untrustworthy) declared size. + $collection = new Document([ + '$id' => ID::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'text', + 'type' => Database::VAR_TEXT, + 'format' => '', + 'size' => 1048576, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure($collection, Database::VAR_INTEGER); + + $base = [ + '$collection' => ID::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + // A TEXT column is limited to 65,535 bytes. Validation measures the + // value's actual byte length, so a value over that capacity is rejected + // even though the declared $size (1MB) would allow the character count. + $tooBig = \str_repeat('a', 65536); + $this->assertEquals(false, $validator->isValid(new Document($base + ['text' => $tooBig]))); + $this->assertEquals('Invalid document structure: Attribute "text" has invalid type. Value must be a valid string no longer than 65535 bytes', $validator->getDescription()); + } + + public function testTextByteSafeValidationMultibyte(): void + { + $collection = new Document([ + '$id' => ID::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'text', + 'type' => Database::VAR_TEXT, + 'format' => '', + 'size' => 1048576, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure($collection, Database::VAR_INTEGER); + + $base = [ + '$collection' => ID::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + // Multi-byte content over the byte capacity is rejected the same way + // (20,000 emoji = 80,000 bytes in utf8mb4). + $multibyte = \str_repeat('📝', 20000); + $this->assertEquals(false, $validator->isValid(new Document($base + ['text' => $multibyte]))); + } + + public function testTextByteSafeValidationValid(): void + { + $collection = new Document([ + '$id' => ID::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'text', + 'type' => Database::VAR_TEXT, + 'format' => '', + 'size' => 1048576, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure($collection, Database::VAR_INTEGER); + + $base = [ + '$collection' => ID::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + // A value that fills the column's full byte capacity is accepted. + $ok = \str_repeat('a', 65535); + $this->assertEquals(true, $validator->isValid(new Document($base + ['text' => $ok]))); + } + public function testArrayOfStringsValidation(): void { $validator = new Structure( @@ -959,7 +1069,26 @@ public function testTextValidation(): void 'published' => true, 'tags' => ['dog', 'cat', 'mouse'], 'feedback' => 'team@appwrite.io', - 'text_field' => \str_repeat('a', 65535), + 'text_field' => \str_repeat('a', 65535), // fills the TEXT column's full byte capacity + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00' + ]))); + + // 'text_field' is optional (required => false). Structure short-circuits + // on `$required === false && is_null($value)` and `continue`s before + // building or running any validators, so the null never reaches the + // ByteLength check (which would otherwise reject it, since + // is_string(null) is false). Hence a null optional text value passes. + $this->assertEquals(true, $validator->isValid(new Document([ + '$collection' => ID::custom('posts'), + 'title' => 'Demo Title', + 'description' => 'Demo description', + 'rating' => 5, + 'price' => 1.99, + 'published' => true, + 'tags' => ['dog', 'cat', 'mouse'], + 'feedback' => 'team@appwrite.io', + 'text_field' => null, '$createdAt' => '2000-04-01T12:00:00.000+00:00', '$updatedAt' => '2000-04-01T12:00:00.000+00:00' ]))); @@ -978,7 +1107,7 @@ public function testTextValidation(): void '$updatedAt' => '2000-04-01T12:00:00.000+00:00' ]))); - $this->assertEquals('Invalid document structure: Attribute "text_field" has invalid type. Value must be a valid string and no longer than 65535 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "text_field" has invalid type. Value must be a valid string no longer than 65535 bytes', $validator->getDescription()); $this->assertEquals(false, $validator->isValid(new Document([ '$collection' => ID::custom('posts'), @@ -994,7 +1123,7 @@ public function testTextValidation(): void '$updatedAt' => '2000-04-01T12:00:00.000+00:00' ]))); - $this->assertEquals('Invalid document structure: Attribute "text_field" has invalid type. Value must be a valid string and no longer than 65535 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "text_field" has invalid type. Value must be a valid string no longer than 65535 bytes', $validator->getDescription()); } public function testMediumtextValidation(): void @@ -1032,7 +1161,58 @@ public function testMediumtextValidation(): void '$updatedAt' => '2000-04-01T12:00:00.000+00:00' ]))); - $this->assertEquals('Invalid document structure: Attribute "mediumtext_field" has invalid type. Value must be a valid string and no longer than 16777215 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "mediumtext_field" has invalid type. Value must be a valid string no longer than 16777215 bytes', $validator->getDescription()); + } + + public function testMediumtextSizedValidation(): void + { + // A mediumtext attribute with a declared size of 100. The declared size + // is enforced as a byte limit (ByteLength), independent of the column's + // physical 16MB ceiling. + $collection = new Document([ + '$id' => ID::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'mediumtext', + 'type' => Database::VAR_MEDIUMTEXT, + 'format' => '', + 'size' => 100, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure($collection, Database::VAR_INTEGER); + + $base = [ + '$collection' => ID::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + // A value of exactly 100 bytes fits the declared size. + $exact = \str_repeat('a', 100); + $this->assertEquals(true, $validator->isValid(new Document($base + ['mediumtext' => $exact]))); + + // 101 bytes exceeds the declared size of 100 bytes and is rejected. + $tooBig = \str_repeat('a', 101); + $this->assertEquals(false, $validator->isValid(new Document($base + ['mediumtext' => $tooBig]))); + $this->assertEquals('Invalid document structure: Attribute "mediumtext" has invalid type. Value must be a valid string no longer than 100 bytes', $validator->getDescription()); + + // Each '📝' is 4 bytes in utf8mb4, so 25 chars = 100 bytes fits exactly, + // while 26 chars = 104 bytes exceeds the declared 100-byte size. + $exactMultibyte = \str_repeat('📝', 25); + $this->assertEquals(true, $validator->isValid(new Document($base + ['mediumtext' => $exactMultibyte]))); + + $tooBigMultibyte = \str_repeat('📝', 26); + $this->assertEquals(false, $validator->isValid(new Document($base + ['mediumtext' => $tooBigMultibyte]))); + $this->assertEquals('Invalid document structure: Attribute "mediumtext" has invalid type. Value must be a valid string no longer than 100 bytes', $validator->getDescription()); } public function testLongtextValidation(): void @@ -1070,7 +1250,7 @@ public function testLongtextValidation(): void '$updatedAt' => '2000-04-01T12:00:00.000+00:00' ]))); - $this->assertEquals('Invalid document structure: Attribute "longtext_field" has invalid type. Value must be a valid string and no longer than 4294967295 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "longtext_field" has invalid type. Value must be a valid string no longer than 4294967295 bytes', $validator->getDescription()); } public function testStringTypeArrayValidation(): void