diff --git a/lang/php/lib/DataFile/AvroDataIODecompressionSizeException.php b/lang/php/lib/DataFile/AvroDataIODecompressionSizeException.php new file mode 100644 index 00000000000..5748f48b389 --- /dev/null +++ b/lang/php/lib/DataFile/AvroDataIODecompressionSizeException.php @@ -0,0 +1,40 @@ + object container metadata @@ -220,18 +234,69 @@ private function readBlockHeader(): string|int return $this->decoder->readLong(); } + /** + * The maximum number of bytes a single block is allowed to decompress to. + */ + private static function maxDecompressLength(): int + { + $value = getenv(self::MAX_DECOMPRESS_LENGTH_ENV); + if (false !== $value && ctype_digit($value) && (int) $value > 0) { + return (int) $value; + } + + return self::DEFAULT_MAX_DECOMPRESS_LENGTH; + } + + /** + * @throws AvroDataIODecompressionSizeException if the length exceeds the limit + */ + private static function checkDecompressLength(int $length, int $maxLength): void + { + if ($length > $maxLength) { + throw new AvroDataIODecompressionSizeException($maxLength); + } + } + /** * @throws AvroException */ private function gzUncompress(string $compressed): string { - $datum = gzinflate($compressed); + $maxLength = self::maxDecompressLength(); + $context = inflate_init(ZLIB_ENCODING_RAW); + if (false === $context) { + throw new AvroException('deflate uncompression failed.'); + } - if (false === $datum) { - throw new AvroException('gzip uncompression failed.'); + // Inflate in chunks and check the running total after each step so an + // over-large (or malicious) block is rejected before its whole output is + // accumulated (each inflated chunk is still materialized), while genuine + // decompression errors (inflate_add === false) are reported distinctly. + // Pieces are collected and joined once at the end to avoid repeatedly + // reallocating a growing result string. + $pieces = []; + $total = 0; + $length = strlen($compressed); + for ($offset = 0; $offset < $length; $offset += self::INFLATE_CHUNK_SIZE) { + $piece = substr($compressed, $offset, self::INFLATE_CHUNK_SIZE); + $out = inflate_add($context, $piece); + if (false === $out) { + throw new AvroException('deflate uncompression failed.'); + } + $pieces[] = $out; + $total += strlen($out); + self::checkDecompressLength($total, $maxLength); } - return $datum; + $out = inflate_add($context, '', ZLIB_FINISH); + if (false === $out) { + throw new AvroException('deflate uncompression failed.'); + } + $pieces[] = $out; + $total += strlen($out); + self::checkDecompressLength($total, $maxLength); + + return implode('', $pieces); } /** @@ -248,6 +313,8 @@ private function zstdUncompress(string $compressed): string throw new AvroException('zstd uncompression failed.'); } + self::checkDecompressLength(strlen($datum), self::maxDecompressLength()); + return $datum; } @@ -265,6 +332,8 @@ private function bzUncompress(string $compressed): string throw new AvroException('bz2 uncompression failed.'); } + self::checkDecompressLength(strlen($datum), self::maxDecompressLength()); + return $datum; } @@ -276,17 +345,77 @@ private function snappyUncompress(string $compressed): string if (!extension_loaded('snappy')) { throw new AvroException('Please install ext-snappy to use snappy compression.'); } - $crc32 = unpack('N', substr((string) $compressed, -4))[1]; - $datum = snappy_uncompress(substr((string) $compressed, 0, -4)); + $maxLength = self::maxDecompressLength(); + // The block is a Snappy payload followed by a 4-byte CRC32 trailer; a + // shorter block is malformed and must not be sliced with negative + // offsets (which would make unpack() return false below). + if (strlen($compressed) < 4) { + throw new AvroException('snappy uncompression failed - block too small.'); + } + $payload = substr($compressed, 0, -4); + $unpacked = unpack('N', substr($compressed, -4)); + if (false === $unpacked) { + throw new AvroException('snappy uncompression failed - missing crc32 trailer.'); + } + $crc32 = $unpacked[1]; + // The Snappy block header declares the uncompressed length as a varint; + // reject an over-large block before allocating for it. Parsed with an + // early cap so it stays correct even if a 32-bit int would overflow. + self::ensureSnappyWithinLimit($payload, $maxLength); + $datum = snappy_uncompress($payload); if (false === $datum) { throw new AvroException('snappy uncompression failed.'); } - if ($crc32 !== crc32($datum)) { + self::checkDecompressLength(strlen($datum), $maxLength); + + // Compare the CRC32 values in a common unsigned representation: + // unpack('N') can yield a float (> PHP_INT_MAX) on 32-bit PHP while + // crc32() yields a (possibly negative) int, so a strict/int comparison + // would report false mismatches for valid data. + if (sprintf('%u', $crc32) !== sprintf('%u', crc32($datum))) { throw new AvroException('snappy uncompression failed - crc32 mismatch.'); } return $datum; } + + /** + * Reject a Snappy block whose declared uncompressed length (a little-endian + * base-128 varint at the start of the block) exceeds $maxLength, before + * allocating for it. The running length is compared against the cap after + * every group, and any wrap to a negative value (32-bit int overflow) is + * treated as over the limit, so the guard holds on 32-bit builds too. A + * varint longer than five bytes is malformed and is rejected as well. + * + * @throws AvroException if the declared length exceeds the limit or is malformed + */ + private static function ensureSnappyWithinLimit(string $data, int $maxLength): void + { + $result = 0; + $shift = 0; + $length = strlen($data); + for ($i = 0; $i < $length; $i++) { + $byte = ord($data[$i]); + $result += ($byte & 0x7F) << $shift; + if ($result < 0 || $result > $maxLength) { + throw new AvroDataIODecompressionSizeException($maxLength); + } + if (0 === ($byte & 0x80)) { + return; // declared length is within the limit + } + $shift += 7; + if ($shift > 28) { + // A Snappy uncompressed length is a uint32, encoded in at most + // five varint bytes; a longer encoding is malformed. + throw new AvroException('snappy uncompression failed - malformed length header.'); + } + } + + // The data ended before a terminating byte (top bit clear) was seen, so + // the length header is truncated/malformed; reject it rather than + // letting a malformed block bypass the guard. + throw new AvroException('snappy uncompression failed - truncated length header.'); + } } diff --git a/lang/php/lib/autoload.php b/lang/php/lib/autoload.php index de1a863513b..c563dc07243 100644 --- a/lang/php/lib/autoload.php +++ b/lang/php/lib/autoload.php @@ -28,6 +28,7 @@ include __DIR__.'/DataFile/AvroDataIO.php'; include __DIR__.'/DataFile/AvroDataIOException.php'; +include __DIR__.'/DataFile/AvroDataIODecompressionSizeException.php'; include __DIR__.'/DataFile/AvroDataIOReader.php'; include __DIR__.'/DataFile/AvroDataIOWriter.php'; diff --git a/lang/php/test/DataFileTest.php b/lang/php/test/DataFileTest.php index 7810c593a78..880e7183473 100644 --- a/lang/php/test/DataFileTest.php +++ b/lang/php/test/DataFileTest.php @@ -23,6 +23,8 @@ namespace Apache\Avro\Tests; use Apache\Avro\DataFile\AvroDataIO; +use Apache\Avro\DataFile\AvroDataIODecompressionSizeException; +use Apache\Avro\DataFile\AvroDataIOReader; use PHPUnit\Framework\TestCase; class DataFileTest extends TestCase @@ -386,6 +388,94 @@ public function test_differing_schemas_with_complex_objects(): void } } + /** + * A block with a very high compression ratio can expand to far more memory + * than its compressed size; reading such a block must be rejected once its + * decompressed size would exceed the configured maximum. + */ + public function test_deflate_block_decompression_limit(): void + { + $data_file = $this->add_data_file('data-decompress-limit-deflate.avr'); + $dw = AvroDataIO::openFile($data_file, 'w', '"string"', AvroDataIO::DEFLATE_CODEC); + $dw->append(str_repeat('a', 64 * 1024)); // 64 KiB, compresses tiny + $dw->close(); + + $previous = getenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'=1024'); + + try { + $dr = AvroDataIO::openFile($data_file); + + try { + $dr->data(); + $this->fail('expected a decompression size exception'); + } catch (AvroDataIODecompressionSizeException $e) { + // expected + } finally { + $dr->close(); + } + } finally { + if (false === $previous) { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + } else { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'='.$previous); + } + } + } + + public function test_deflate_block_within_decompression_limit(): void + { + $data_file = $this->add_data_file('data-decompress-within-limit.avr'); + $payload = 'hello world'; + $dw = AvroDataIO::openFile($data_file, 'w', '"string"', AvroDataIO::DEFLATE_CODEC); + $dw->append($payload); + $dw->close(); + + $previous = getenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'=1048576'); + + try { + $dr = AvroDataIO::openFile($data_file); + + try { + $data = $dr->data(); + $this->assertSame([$payload], $data); + } finally { + $dr->close(); + } + } finally { + if (false === $previous) { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + } else { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'='.$previous); + } + } + } + + public function test_snappy_block_decompression_limit(): void + { + if (!extension_loaded('snappy')) { + $this->markTestSkipped('snappy extension not available'); + } + $this->assertCodecRejectsOversizedBlock(AvroDataIO::SNAPPY_CODEC); + } + + public function test_zstandard_block_decompression_limit(): void + { + if (!extension_loaded('zstd')) { + $this->markTestSkipped('zstd extension not available'); + } + $this->assertCodecRejectsOversizedBlock(AvroDataIO::ZSTANDARD_CODEC); + } + + public function test_bzip2_block_decompression_limit(): void + { + if (!extension_loaded('bz2')) { + $this->markTestSkipped('bz2 extension not available'); + } + $this->assertCodecRejectsOversizedBlock(AvroDataIO::BZIP2_CODEC); + } + protected function add_data_file(string $data_file): string { $data_file = "$data_file.".self::current_timestamp(); @@ -411,4 +501,38 @@ protected static function remove_data_file($data_file): void unlink($data_file); } } + + /** + * Write a single, highly compressible block with the given codec, then read + * it back with a small decompression limit and assert it is rejected. + */ + private function assertCodecRejectsOversizedBlock(string $codec): void + { + $data_file = $this->add_data_file(sprintf('data-decompress-limit-%s.avr', $codec)); + $dw = AvroDataIO::openFile($data_file, 'w', '"string"', $codec); + $dw->append(str_repeat('a', 64 * 1024)); // 64 KiB, compresses tiny + $dw->close(); + + $previous = getenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'=1024'); + + try { + $dr = AvroDataIO::openFile($data_file); + + try { + $dr->data(); + $this->fail(sprintf('expected a decompression size exception for %s', $codec)); + } catch (AvroDataIODecompressionSizeException $e) { + // expected + } finally { + $dr->close(); + } + } finally { + if (false === $previous) { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV); + } else { + putenv(AvroDataIOReader::MAX_DECOMPRESS_LENGTH_ENV.'='.$previous); + } + } + } }