Merge pull request #111 from j0k3r/fix/tidy-php85

Fix bad encoding for `tidy_repair_string`
Add PHP 8.5 on CI
2026-09-27 06:26:17 +00:00 · 2026-03-27 20:34:19 +01:00 · 2026-03-27 20:31:52 +01:00 · 2026-03-27 20:29:09 +01:00 · 2025-06-03 09:22:21 +02:00 · 2025-05-28 01:10:59 +02:00
3 changed files with 33 additions and 39 deletions
@@ -25,6 +25,7 @@ jobs:
          - "8.2"
          - "8.3"
          - "8.4"
+          - "8.5"

    steps:
      - name: "Checkout"
@@ -88,7 +88,7 @@ class Readability implements LoggerAwareInterface
        'enclose-text' => true,
        'merge-divs' => true,
        // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
        'output-encoding' => 'utf8',
        'hide-comments' => true,
    ];
@@ -1118,7 +1118,7 @@ class Readability implements LoggerAwareInterface

        $topCandidates = array_filter(
            $topCandidates,
-            fn ($v, $idx) => 0 === $idx || null !== $v,
+            static fn ($v, $idx) => 0 === $idx || null !== $v,
            \ARRAY_FILTER_USE_BOTH
        );
        $topCandidate = $topCandidates[0];
@@ -1422,7 +1422,7 @@ class Readability implements LoggerAwareInterface
            unset($tidy);
        }

-        $this->html = self::ensureMetaCharset((string) $this->html);
+        $this->html = self::entitizeNonAscii((string) $this->html);

        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
            $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1481,7 +1481,7 @@ class Readability implements LoggerAwareInterface
    private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
    {
        $childNodes = iterator_to_array($node->childNodes);
-        $children = array_filter($childNodes, fn ($childNode) => $childNode instanceof \DOMElement);
+        $children = array_filter($childNodes, static fn ($childNode) => $childNode instanceof \DOMElement);

        // There should be exactly 1 element child with given tag
        if (1 !== \count($children) || $children[0]->nodeName !== $tag) {
@@ -1512,43 +1512,19 @@ class Readability implements LoggerAwareInterface
    }

    /**
-     * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document.
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
     *
     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
-     * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag.
     *
     * @param string $html UTF-8 encoded document
     */
-    private static function ensureMetaCharset(string $html): string
+    private static function entitizeNonAscii(string $html): string
    {
-        $charsetTag = '<meta charset="utf-8">';
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];

-        // Only look at first 1024 bytes since, according to HTML5 specification,
-        // that’s where <meta> elements declaring a character encoding must be located.
-        // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset
-        $start = substr($html, 0, 1000);
-
-        if (1 === preg_match('/<meta[^>]+charset/i', $start)) {
-            // <meta> tag is already present, no need for modification.
-            return $html;
-        }
-
-        if (1 === preg_match('/<head[^>]*>/i', $start)) {
-            // <head> tag was located, <meta> tags go there.
-            $html = preg_replace('/<head[^>]*>/i', '$0' . $charsetTag, $html, 1);
-
-            return $html;
-        }
-
-        if (1 === preg_match('/<html[^>]*>/i', $start)) {
-            // <html> tag was located, let’s put it inside and have parser create <head>.
-            $html = preg_replace('/<html[^>]*>/i', '$0' . $charsetTag, $html, 1);
-
-            return $html;
-        }
-
-        // Fallback – just plop the <meta> at the start of the fragment.
-        return $charsetTag . $html;
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
    }
 }
@@ -21,6 +21,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
    public function testConstructDefault(): void
    {
        $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);
        $readability->init();

        $this->assertNull($readability->url);
@@ -323,7 +324,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $oldErrorReporting = error_reporting(\E_ALL);
        $oldDisplayErrors = ini_set('display_errors', '1');
        // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
+        set_error_handler(static function (int $errno, string $errstr, string $errfile, int $errline) {
            throw new \Exception($errstr, $errno);
        });

@@ -529,6 +530,20 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        }
    }

+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle(): void
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
    /**
     * @return array<string, array{0: string, 1: string, 2?: bool}>
     */
@@ -536,21 +551,21 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
    {
        return [
            'meta' => [
-                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                'fr',
            ],
            'head' => [
-                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                'fr',
            ],
            'headless' => [
-                '<html lang="fr"><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                'fr',
                // tidy would add <head> tag.
                false,
            ],
            'fragment' => [
-                '<article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article>',
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
                '',
                // tidy would add <html>.
                false,
@@ -569,6 +584,8 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $this->assertTrue($res);
        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
    }

    private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
Author	SHA1	Message	Date
Jérémy BenoistandGitHub	03533f5e4f	Merge pull request #111 from j0k3r/fix/tidy-php85 Fix bad encoding for `tidy_repair_string`	2026-03-27 20:34:19 +01:00
Jeremy Benoist	009b4ab9b8	Add PHP 8.5 on CI	2026-03-27 20:31:52 +01:00
Jeremy Benoist	f9e73fb49a	Fix bad encoding for `tidy_repair_string` Tidy on PHP 8.5 is more restrictive on what can be given as input encoding. Before, it worked as an unknown value was converted to `utf8`.	2026-03-27 20:29:09 +01:00
Jérémy BenoistandGitHub	3042990efc	Merge pull request #106 from jtojnar/encode Fix character decoding regression when `title` precedes `meta[charset]`	2025-06-03 09:22:21 +02:00
Jan Tojnar	8b89d70b1a	Fix character decoding regression when `title` precedes `meta[charset]` Because of PHP 8.2 deprecation, in `f14428e4c0`, we stopped converting non-ASCII characters to HTML entities. Instead, we started to explicitly insert `meta[charset]` tag at the start of the document. Later, we discovered that was breaking `html[lang]` so, in `efbbc86df9`, we made the insertion smarter. One of the improvements was that it would not insert the `meta[charset]` tag when it was already present. That, however, broke websites that had `title` tag before `meta[charset]`. On those, libxml2 would decode the `title` contents as ISO-8859-1. We could improve the logic (e.g. check that there is not text content before `meta[charset]`) or insert the tag unconditionally but it will probably be simplest to just go back to converting the non-ASCII characters to entities, just using non-deprecated function variant.	2025-05-28 01:10:59 +02:00
Jan Tojnar	3e9b15db46	tests: Check encoding was preserved in `testHtmlLang` The fix introduced in `efbbc86df9` alongside this test also manipulates `meta[charset]` but we were not checking if it does not break encoding.	2025-05-28 00:51:49 +02:00