From 40219d459507d88f95f94f65ad183cbbfcb7c74d Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 28 May 2025 01:10:59 +0200 Subject: [PATCH] Fix character decoding regression when `title` precedes `meta[charset]` Because of PHP 8.2 deprecation, in f14428e4c0fa34b28f6b8a7696e62790bcde363e, we stopped converting non-ASCII characters to HTML entities. Instead, we started to explicitly insert `meta[charset]` tag at the start of the document. Later, we discovered that was breaking `html[lang]` so, in efbbc86df9716a3ab1ed8a351d9e8316f3a2aab0, we made the insertion smarter. One of the improvements was that it would not insert the `meta[charset]` tag when it was already present. That, however, broke websites that had `title` tag before `meta[charset]`. On those, libxml2 would decode the `title` contents as ISO-8859-1. We could improve the logic (e.g. check that there is not text content before `meta[charset]`) or insert the tag unconditionally but it will probably be simplest to just go back to converting the non-ASCII characters to entities, just using non-deprecated function variant. --- src/Readability.php | 38 +++++++------------------------------- tests/ReadabilityTest.php | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/Readability.php b/src/Readability.php index 05a6c9f..d6fcbe2 100644 --- a/src/Readability.php +++ b/src/Readability.php @@ -1435,7 +1435,7 @@ class Readability implements LoggerAwareInterface unset($tidy); } - $this->html = self::ensureMetaCharset((string) $this->html); + $this->html = self::entitizeNonAscii((string) $this->html); if ('html5lib' === $this->parser || 'html5' === $this->parser) { $this->dom = (new HTML5())->loadHTML($this->html); @@ -1455,43 +1455,19 @@ class Readability implements LoggerAwareInterface } /** - * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document. + * Converts non-ASCII UTF-8 characters to numeric HTML entities. * * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag. * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding. - * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag. * * @param string $html UTF-8 encoded document */ - private static function ensureMetaCharset($html) + private static function entitizeNonAscii($html) { - $charsetTag = ''; + $convmap = [ + 0x80, 0x1FFFFF, 0, 0x10FFFF, + ]; - // Only look at first 1024 bytes since, according to HTML5 specification, - // that’s where elements declaring a character encoding must be located. - // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset - $start = substr($html, 0, 1000); - - if (1 === preg_match('/]+charset/i', $start)) { - // tag is already present, no need for modification. - return $html; - } - - if (1 === preg_match('/]*>/i', $start)) { - // tag was located, tags go there. - $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1); - - return $html; - } - - if (1 === preg_match('/]*>/i', $start)) { - // tag was located, let’s put it inside and have parser create . - $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1); - - return $html; - } - - // Fallback – just plop the at the start of the fragment. - return $charsetTag . $html; + return mb_encode_numericentity($html, $convmap, 'utf8', true); } } diff --git a/tests/ReadabilityTest.php b/tests/ReadabilityTest.php index 0d753ab..df8a13b 100644 --- a/tests/ReadabilityTest.php +++ b/tests/ReadabilityTest.php @@ -486,6 +486,20 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase $this->assertStringContainsString('Down the Rabbit-Hole', $readability->getContent()->getInnerHtml()); } + // https://github.com/wallabag/wallabag/issues/8158 + public function testCharsetAfterTitle() + { + $readability = $this->getReadability('Tõde ja õigus I

See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.

', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I'); + $readability->convertLinksToFootnotes = true; + $res = $readability->init(); + + $this->assertTrue($res); + $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent()); + $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle()); + $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml()); + $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml()); + } + /** * @return array */