Merge pull request #131 from j0k3r/dependabot/composer/rector/rector-2.5.7

Updates the requirements on [friendsofphp/php-cs-fixer](https://github.com/PHP-CS-Fixer/PHP-CS-Fixer) to permit the latest version.
- [Release notes](https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/releases)
- [Changelog](https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/blob/master/CHANGELOG.md)
- [Commits](https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/compare/v3.95.12...v3.95.15)

---
updated-dependencies:
- dependency-name: friendsofphp/php-cs-fixer
  dependency-version: 3.95.15
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "composer"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/continuous-integration.yml

@@ -25,6 +25,7 @@ jobs:
           - "8.2"
           - "8.3"
           - "8.4"
+          - "8.5"
 
     steps:
       - name: "Checkout"

.php-cs-fixer.php

@@ -30,6 +30,8 @@ return (new PhpCsFixer\Config())
         'modernize_strpos' => false,
         // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
         'no_null_property_initialization' => false,
+        // @Symfony:risky started removing them but we will probably want to go the opposite direction.
+        'declare_strict_types' => false,
     ])
     ->setFinder($finder)
 ;

composer.json

@@ -30,12 +30,12 @@
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
-        "friendsofphp/php-cs-fixer": "^3.0",
+        "friendsofphp/php-cs-fixer": "3.95.15",
         "monolog/monolog": "^1.24|^2.1",
         "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0",
-        "phpstan/phpstan": "^2.0",
-        "phpstan/phpstan-phpunit": "^2.0",
-        "rector/rector": "^2.0.0"
+        "phpstan/phpstan": "2.2.5",
+        "phpstan/phpstan-phpunit": "2.0.18",
+        "rector/rector": "2.5.7"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -46,6 +46,9 @@
     "autoload-dev": {
         "psr-4": { "Tests\\Readability\\": "tests/" }
     },
+    "config": {
+        "lock": false
+    },
     "scripts": {
         "fix": "php-cs-fixer fix --verbose --diff",
         "phpstan": "phpstan analyze --memory-limit 512M",

src/JSLikeHTMLElement.php

@@ -43,7 +43,7 @@ class JSLikeHTMLElement extends \DOMElement
      * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
      * ```
      */
-    public function __set($name, $value)
+    public function __set(string $name, string $value): void
     {
         if ('innerHTML' !== $name) {
             $trace = debug_backtrace();
@@ -108,7 +108,7 @@ class JSLikeHTMLElement extends \DOMElement
      * $string = $div->innerHTML;
      * ```
      */
-    public function __get($name)
+    public function __get(string $name): string
     {
         if ('innerHTML' === $name) {
             $inner = '';
@@ -124,20 +124,22 @@ class JSLikeHTMLElement extends \DOMElement
 
         $trace = debug_backtrace();
         trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);
+
+        return '';
     }
 
-    public function __toString()
+    public function __toString(): string
     {
         return '[' . $this->tagName . ']';
     }
 
-    public function getInnerHtml()
+    public function getInnerHtml(): string
     {
         return $this->__get('innerHTML');
     }
 
-    public function setInnerHtml($value)
+    public function setInnerHtml(string $value): void
     {
-        return $this->__set('innerHTML', $value);
+        $this->__set('innerHTML', $value);
     }
 }

src/Readability.php

@@ -88,7 +88,7 @@ class Readability implements LoggerAwareInterface
         'enclose-text' => true,
         'merge-divs' => true,
         // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
         'output-encoding' => 'utf8',
         'hide-comments' => true,
     ];
@@ -213,7 +213,9 @@ class Readability implements LoggerAwareInterface
      */
     public function init(): bool
     {
+        if (!isset($this->dom)) {
             $this->loadHtml();
+        }
 
         if (!isset($this->dom->documentElement)) {
             return false;
@@ -751,6 +753,76 @@ class Readability implements LoggerAwareInterface
         $this->flags &= ~$flag;
     }
 
+    /**
+     * Load HTML in a DOMDocument.
+     * Apply Pre filters
+     * Cleanup HTML using Tidy (or not).
+     */
+    public function loadHtml(): void
+    {
+        $this->original_html = $this->html;
+
+        $this->logger->debug('Parsing URL: ' . $this->url);
+
+        if ($this->url) {
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
+        }
+
+        mb_internal_encoding('UTF-8');
+        mb_http_output('UTF-8');
+        mb_regex_encoding('UTF-8');
+
+        // HACK: dirty cleanup to replace some stuff; shouldn't use regexps with HTML but well...
+        if (!$this->flagIsActive(self::FLAG_DISABLE_PREFILTER)) {
+            foreach ($this->pre_filters as $search => $replace) {
+                $this->html = preg_replace($search, $replace, $this->html);
+            }
+            unset($search, $replace);
+        }
+
+        if ('' === trim($this->html)) {
+            $this->html = '<html></html>';
+        }
+
+        /*
+         * Use tidy (if it exists).
+         * This fixes problems with some sites which would otherwise trouble DOMDocument's HTML parsing.
+         * Although sometimes it makes matters worse, which is why there is an option to disable it.
+         */
+        if ($this->useTidy) {
+            $this->logger->debug('Tidying document');
+
+            $tidy = tidy_repair_string($this->html, $this->tidy_config, 'UTF8');
+            if (false !== $tidy && $this->html !== $tidy) {
+                $this->tidied = true;
+                $this->html = $tidy;
+                $this->html = preg_replace('/[\r\n]+/is', "\n", $this->html);
+            }
+            unset($tidy);
+        }
+
+        $this->html = self::entitizeNonAscii((string) $this->html);
+
+        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
+            $this->dom = (new HTML5())->loadHTML($this->html);
+        }
+
+        if ('libxml' === $this->parser) {
+            libxml_use_internal_errors(true);
+
+            $this->dom = new \DOMDocument();
+            $this->dom->preserveWhiteSpace = false;
+            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
+
+            libxml_use_internal_errors(false);
+        }
+
+        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
+    }
+
     /**
      * Get the article title as an H1.
      *
@@ -1003,7 +1075,11 @@ class Readability implements LoggerAwareInterface
                     }
 
                     if ($this->hasSingleTagInsideElement($node, 'p') && $this->getLinkDensity($node) < 0.25) {
-                        $newNode = $node->childNodes->item(0);
+                        // In some cases when tidy is disabled the first item may not be a DOMElement so we apply a filter
+                        $newNode = array_values(array_filter(
+                            iterator_to_array($node->childNodes),
+                            static fn ($childNode) => $childNode instanceof \DOMElement
+                        ))[0];
                         $node->parentNode->replaceChild($newNode, $node);
                         $nodesToScore[] = $newNode;
                     }
@@ -1118,7 +1194,7 @@ class Readability implements LoggerAwareInterface
 
         $topCandidates = array_filter(
             $topCandidates,
-            fn ($v, $idx) => 0 === $idx || null !== $v,
+            static fn ($v, $idx) => 0 === $idx || null !== $v,
             \ARRAY_FILTER_USE_BOTH
         );
         $topCandidate = $topCandidates[0];
@@ -1371,76 +1447,6 @@ class Readability implements LoggerAwareInterface
         }
     }
 
-    /**
-     * Load HTML in a DOMDocument.
-     * Apply Pre filters
-     * Cleanup HTML using Tidy (or not).
-     */
-    private function loadHtml(): void
-    {
-        $this->original_html = $this->html;
-
-        $this->logger->debug('Parsing URL: ' . $this->url);
-
-        if ($this->url) {
-            $host = parse_url($this->url, \PHP_URL_HOST);
-            if (null !== $host) {
-                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
-            }
-        }
-
-        mb_internal_encoding('UTF-8');
-        mb_http_output('UTF-8');
-        mb_regex_encoding('UTF-8');
-
-        // HACK: dirty cleanup to replace some stuff; shouldn't use regexps with HTML but well...
-        if (!$this->flagIsActive(self::FLAG_DISABLE_PREFILTER)) {
-            foreach ($this->pre_filters as $search => $replace) {
-                $this->html = preg_replace($search, $replace, $this->html);
-            }
-            unset($search, $replace);
-        }
-
-        if ('' === trim($this->html)) {
-            $this->html = '<html></html>';
-        }
-
-        /*
-         * Use tidy (if it exists).
-         * This fixes problems with some sites which would otherwise trouble DOMDocument's HTML parsing.
-         * Although sometimes it makes matters worse, which is why there is an option to disable it.
-         */
-        if ($this->useTidy) {
-            $this->logger->debug('Tidying document');
-
-            $tidy = tidy_repair_string($this->html, $this->tidy_config, 'UTF8');
-            if (false !== $tidy && $this->html !== $tidy) {
-                $this->tidied = true;
-                $this->html = $tidy;
-                $this->html = preg_replace('/[\r\n]+/is', "\n", $this->html);
-            }
-            unset($tidy);
-        }
-
-        $this->html = self::ensureMetaCharset((string) $this->html);
-
-        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
-            $this->dom = (new HTML5())->loadHTML($this->html);
-        }
-
-        if ('libxml' === $this->parser) {
-            libxml_use_internal_errors(true);
-
-            $this->dom = new \DOMDocument();
-            $this->dom->preserveWhiteSpace = false;
-            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
-
-            libxml_use_internal_errors(false);
-        }
-
-        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
-    }
-
     private function getAncestors(\DOMElement $node, int $maxDepth = 0): array
     {
         $ancestors = [];
@@ -1481,10 +1487,17 @@ class Readability implements LoggerAwareInterface
     private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
     {
         $childNodes = iterator_to_array($node->childNodes);
-        $children = array_filter($childNodes, fn ($childNode) => $childNode instanceof \DOMElement);
+        $children = array_filter($childNodes, static fn ($childNode) => $childNode instanceof \DOMElement);
+
+        // array_first() has been added in PHP 8.5, failing back for older versions
+        if (!\function_exists('array_first')) {
+            $firstChild = $children ? $children[array_key_first($children)] : null;
+        } else {
+            $firstChild = array_first($children);
+        }
 
         // There should be exactly 1 element child with given tag
-        if (1 !== \count($children) || $children[0]->nodeName !== $tag) {
+        if (1 !== \count($children) || $firstChild->nodeName !== $tag) {
             return false;
         }
 
@@ -1512,43 +1525,19 @@ class Readability implements LoggerAwareInterface
     }
 
     /**
-     * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document.
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
      *
      * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
      * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
-     * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag.
      *
      * @param string $html UTF-8 encoded document
      */
-    private static function ensureMetaCharset(string $html): string
+    private static function entitizeNonAscii(string $html): string
     {
-        $charsetTag = '<meta charset="utf-8">';
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];
 
-        // Only look at first 1024 bytes since, according to HTML5 specification,
-        // that’s where <meta> elements declaring a character encoding must be located.
-        // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset
-        $start = substr($html, 0, 1000);
-
-        if (1 === preg_match('/<meta[^>]+charset/i', $start)) {
-            // <meta> tag is already present, no need for modification.
-            return $html;
-        }
-
-        if (1 === preg_match('/<head[^>]*>/i', $start)) {
-            // <head> tag was located, <meta> tags go there.
-            $html = preg_replace('/<head[^>]*>/i', '$0' . $charsetTag, $html, 1);
-
-            return $html;
-        }
-
-        if (1 === preg_match('/<html[^>]*>/i', $start)) {
-            // <html> tag was located, let’s put it inside and have parser create <head>.
-            $html = preg_replace('/<html[^>]*>/i', '$0' . $charsetTag, $html, 1);
-
-            return $html;
-        }
-
-        // Fallback – just plop the <meta> at the start of the fragment.
-        return $charsetTag . $html;
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
     }
 }

tests/ReadabilityTest.php

@@ -21,6 +21,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testConstructDefault(): void
     {
         $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);
         $readability->init();
 
         $this->assertNull($readability->url);
@@ -323,7 +324,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $oldErrorReporting = error_reporting(\E_ALL);
         $oldDisplayErrors = ini_set('display_errors', '1');
         // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
+        set_error_handler(static function (int $errno, string $errstr, string $errfile, int $errline) {
             throw new \Exception($errstr, $errno);
         });
 
@@ -463,6 +464,62 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
     }
 
+    public function dataForHasSingleTagInsideElement(): array
+    {
+        return [
+            'single matching tag, no other content' => [
+                '<div><p>Some text</p></div>', 'p', true,
+            ],
+            'single matching tag with comment' => [
+                '<div><!-- comment --><p>Some text</p></div>', 'p', true,
+            ],
+            'whitespace-only text around single matching tag' => [
+                '<div>   <p>Some text</p>   </div>', 'p', true,
+            ],
+            'two matching tags' => [
+                '<div><p>One</p><p>Two</p></div>', 'p', false,
+            ],
+            'non-whitespace text alongside the single tag' => [
+                '<div>Some text<p>One</p></div>', 'p', false,
+            ],
+            'single tag with a different name' => [
+                '<div><span>One</span></div>', 'p', false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHasSingleTagInsideElement
+     */
+    public function testHasSingleTagInsideElement(string $html, string $tag, bool $expected): void
+    {
+        $dom = new \DOMDocument();
+        $dom->loadHTML('<html><body>' . $html . '</body></html>');
+        $node = $dom->getElementsByTagName('div')->item(0);
+
+        $readability = $this->getReadability('', '', 'libxml', false);
+
+        $method = new \ReflectionMethod($readability, 'hasSingleTagInsideElement');
+        if (\PHP_VERSION_ID < 80100) {
+            $method->setAccessible(true);
+        }
+
+        $this->assertSame($expected, $method->invoke($readability, $node, $tag));
+    }
+
+    public function testDivToPElementWithComment(): void
+    {
+        $text = str_repeat('Padded real article text to reach decent length for content scoring threshold here. ', 8);
+        $html = '<div><!-- comment --><code>some code snippet</code> ' . $text . '</div>';
+
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertStringContainsString('some code snippet', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('Padded real article text', $readability->getContent()->getInnerHtml());
+    }
+
     public function testKeepFootnotes(): void
     {
         // from https://www.schreibdichte.de/blog/feed-aggregator-und-spaeter-lesen-dienst-im-team
@@ -529,6 +586,20 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         }
     }
 
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle(): void
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
     /**
      * @return array<string, array{0: string, 1: string, 2?: bool}>
      */
@@ -536,21 +607,21 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     {
         return [
             'meta' => [
-                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                 'fr',
             ],
             'head' => [
-                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                 'fr',
             ],
             'headless' => [
-                '<html lang="fr"><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
                 'fr',
                 // tidy would add <head> tag.
                 false,
             ],
             'fragment' => [
-                '<article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article>',
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
                 '',
                 // tidy would add <html>.
                 false,
@@ -569,6 +640,8 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
         $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
     }
 
     private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability