diff --git a/src/wp-admin/includes/export.php b/src/wp-admin/includes/export.php index a77cb804f0780..557cb056b9f6a 100644 --- a/src/wp-admin/includes/export.php +++ b/src/wp-admin/includes/export.php @@ -246,7 +246,7 @@ function wxr_cdata( $str ) { $str = (string) $str; if ( ! wp_is_valid_utf8( $str ) ) { - $str = utf8_encode( $str ); + $str = _wp_iso_8859_1_to_utf8( $str ); } // $str = ent2ncr(esc_html($str)); $str = '', ']]]]>', $str ) . ']]>'; diff --git a/src/wp-admin/includes/image.php b/src/wp-admin/includes/image.php index 935c613d561e9..42fb5498704c2 100644 --- a/src/wp-admin/includes/image.php +++ b/src/wp-admin/includes/image.php @@ -1049,13 +1049,13 @@ function wp_read_image_metadata( $file ) { foreach ( array( 'title', 'caption', 'credit', 'copyright', 'camera', 'iso' ) as $key ) { if ( $meta[ $key ] && ! wp_is_valid_utf8( $meta[ $key ] ) ) { - $meta[ $key ] = utf8_encode( $meta[ $key ] ); + $meta[ $key ] = _wp_iso_8859_1_to_utf8( $meta[ $key ] ); } } foreach ( $meta['keywords'] as $key => $keyword ) { if ( ! wp_is_valid_utf8( $keyword ) ) { - $meta['keywords'][ $key ] = utf8_encode( $keyword ); + $meta['keywords'][ $key ] = _wp_iso_8859_1_to_utf8( $keyword ); } } diff --git a/src/wp-includes/utf8.php b/src/wp-includes/utf8.php index 6be22a61ff83a..88aa98158d865 100644 --- a/src/wp-includes/utf8.php +++ b/src/wp-includes/utf8.php @@ -134,6 +134,48 @@ function wp_scrub_utf8( $text ) { } endif; +if ( extension_loaded( 'mbstring' ) ) : + /** + * Converts a string from ISO-8859-1 (latin1) to UTF-8. + * + * This is a last resort for text whose encoding is unknown and which has already + * failed to validate as UTF-8. Interpreting those bytes as ISO-8859-1 is a guess: + * it is correct when the text really is ISO-8859-1, and produces mojibake for any + * other single-byte or multi-byte encoding. Call sites which do know the encoding + * of their text should decode it with {@see \mb_convert_encoding()} instead. + * + * Every byte maps to a code point, so the return value is always valid UTF-8. + * + * This function exists so that the call sites which historically relied on PHP’s + * `utf8_encode()` can keep their behavior after that function’s removal in PHP 9.0. + * It is not a replacement for {@see \wp_scrub_utf8()}, which neutralizes invalid + * bytes instead of reinterpreting them. + * + * @ignore + * @private + * + * @since 7.1.0 + * + * @param string $text Text treated as ISO-8859-1 (latin1) bytes. + * @return string Text converted into UTF-8. + */ + function _wp_iso_8859_1_to_utf8( $text ) { + return mb_convert_encoding( (string) $text, 'UTF-8', 'ISO-8859-1' ); + } +else : + /** + * Fallback function for converting ISO-8859-1 into UTF-8. + * + * @ignore + * @private + * + * @since 7.1.0 + */ + function _wp_iso_8859_1_to_utf8( $text ) { + return _wp_utf8_encode_fallback( $text ); + } +endif; + /** * Returns whether the given string contains Unicode noncharacters. * diff --git a/tests/phpunit/tests/admin/exportWp.php b/tests/phpunit/tests/admin/exportWp.php index f17ef0d4ad343..d8c0dbcef4417 100644 --- a/tests/phpunit/tests/admin/exportWp.php +++ b/tests/phpunit/tests/admin/exportWp.php @@ -475,6 +475,94 @@ public function test_export_with_null_term_meta_values() { $this->assertGreaterThan( 0, count( $xml->channel->item ), 'Export should contain items' ); } + /** + * Ensures the WXR export converts non-UTF-8 text into UTF-8. + * + * The export declares itself as UTF-8, so `wxr_cdata()` has to hand back a valid + * UTF-8 string. Text which fails to validate is decoded as ISO-8859-1 (latin1), + * which is a guess: correct when the text really is latin1, mojibake otherwise. + * Core has no encoding declaration to consult here, so the guess stands. + * + * These cases pin that behavior so it cannot change silently. + * + * @ticket 65828 + * + * @dataProvider data_non_utf8_strings + * + * @param string $input Bytes which are not valid UTF-8. + * @param string $expected Expected CDATA contents. + */ + public function test_wxr_cdata_converts_non_utf8_text( $input, $expected ) { + // Running an export defines the nested WXR helper functions. + $this->get_the_export( array( 'content' => 'post' ) ); + + $actual = wxr_cdata( $input ); + $inner = substr( $actual, strlen( '' ) ); + + $this->assertSame( + $expected, + $inner, + 'Non-UTF-8 bytes should be decoded as ISO-8859-1.' + ); + $this->assertTrue( + wp_is_valid_utf8( $inner ), + 'The exported CDATA section should always contain valid UTF-8.' + ); + } + + /** + * Data provider. + * + * @return array[] + */ + public function data_non_utf8_strings() { + return array( + // Text which is well-formed, but in some encoding other than UTF-8. + 'ISO-8859-1 text' => array( + mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), + 'Café', + ), + 'ISO-8859-2 text' => array( + mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + 'wyró¿nij', + ), + 'Windows-1251 text' => array( + mb_convert_encoding( 'Привет', 'Windows-1251', 'UTF-8' ), + 'Ïðèâåò', + ), + 'Windows-1252 quotations' => array( + mb_convert_encoding( '“quoted”', 'Windows-1252', 'UTF-8' ), + "\u{0093}quoted\u{0094}", + ), + + // Malformed UTF-8. + 'Never-valid byte' => array( "a\xC0b", 'aÀb' ), + 'Truncated sequence' => array( "a\xE2\x9Cb", "aâ\u{009C}b" ), + 'Overlong sequence' => array( "a\xC1\xBFb", 'aÁ¿b' ), + 'Surrogate half' => array( "a\xED\xA0\x80b", "a\u{00ED}\u{00A0}\u{0080}b" ), + + /* + * The guard inspects the whole string, so a single stray byte sends the + * valid portions through the conversion as well and double-encodes them. + */ + 'Valid UTF-8 with one stray byte' => array( "Pi\xC3\xB1a \xC0", 'Piña À' ), + ); + } + + /** + * Ensures valid UTF-8, including multibyte text, survives the export untouched. + * + * @ticket 65828 + */ + public function test_wxr_cdata_preserves_valid_utf8() { + $this->get_the_export( array( 'content' => 'post' ) ); + + $valid = 'Это комментарий. / Βλέπετε ένα σχόλιο. / 🅰'; + $inner = substr( wxr_cdata( $valid ), strlen( '' ) ); + + $this->assertSame( $valid, $inner, 'Valid UTF-8 should pass through unchanged.' ); + } + /** * Ensure that posts types with 'can_export' set to false are not included in the export. * diff --git a/tests/phpunit/tests/image/meta.php b/tests/phpunit/tests/image/meta.php index b6a1849fae9ae..84dfd6b21c29c 100644 --- a/tests/phpunit/tests/image/meta.php +++ b/tests/phpunit/tests/image/meta.php @@ -149,6 +149,96 @@ public function test_utf8_iptc_tags() { $this->assertSame( 'This is a comment. / Это комментарий. / Βλέπετε ένα σχόλιο.', $out['caption'] ); } + /** + * Ensures image metadata is always returned as valid UTF-8. + * + * Core does not read the IPTC coded character set, so the encoding of these fields + * is unknown. Fields which fail to validate as UTF-8 are decoded as ISO-8859-1 + * (latin1): correct when the text really is latin1, mojibake otherwise. Invalid + * UTF-8 reaching the database is stripped by `strip_invalid_text()`, which is why + * the conversion has to happen at all. See #35316. + * + * These cases pin that behavior so it cannot change silently. + * + * @ticket 65828 + * @ticket 35316 + */ + public function test_iptc_non_utf8_text_is_converted() { + if ( ! is_callable( 'iptcembed' ) || ! is_callable( 'iptcparse' ) ) { + $this->markTestSkipped( 'The iptcembed() and iptcparse() functions are required.' ); + } + + $block = $this->build_iptc_block( + array( + 105 => mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + 120 => mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), + 110 => "Credit \xC0", + 116 => "Copyright \xE9", + 25 => array( + 'valid keyword', + "sunset\xC0", + // The Slovak keywords from the image attached to #35316. + mb_convert_encoding( 'Vodná elektráreň Gabčíkovo', 'ISO-8859-2', 'UTF-8' ), + ), + ) + ); + + $file = wp_tempnam( 'iptc-non-utf8.jpg' ); + file_put_contents( $file, iptcembed( $block, DIR_TESTDATA . '/images/test-image.jpg' ) ); + + $out = wp_read_image_metadata( $file ); + + unlink( $file ); + + $this->assertIsArray( $out, 'Metadata should have been read from the image.' ); + + foreach ( array( 'title', 'caption', 'credit', 'copyright' ) as $key ) { + $this->assertTrue( + wp_is_valid_utf8( $out[ $key ] ), + "The '{$key}' field should always be valid UTF-8." + ); + } + + $this->assertSame( + 'Café', + $out['caption'], + 'ISO-8859-1 text should round-trip unchanged.' + ); + + $this->assertSame( + 'wyró¿nij', + $out['title'], + 'Text in another single-byte encoding should be decoded as ISO-8859-1.' + ); + + $this->assertSame( + array( 'valid keyword', 'sunsetÀ', 'Vodná elektráreò Gabèíkovo' ), + $out['keywords'], + 'Keywords should be converted individually while valid entries are left alone.' + ); + } + + /** + * Builds a raw IPTC APP13 block for the given record 2 datasets. + * + * @param array $tags Map of dataset number to a string value or list of string values. + * @return string Binary IPTC block suitable for iptcembed(). + */ + private function build_iptc_block( array $tags ) { + $block = ''; + + foreach ( $tags as $dataset => $values ) { + foreach ( (array) $values as $value ) { + $length = strlen( $value ); + $block .= chr( 0x1C ) . chr( 2 ) . chr( $dataset ) + . chr( ( $length >> 8 ) & 0xFF ) . chr( $length & 0xFF ) + . $value; + } + } + + return $block; + } + /** * wp_read_image_metadata() should return false if the image file doesn't exist. */