Merge pull request #113 from Kdecherf/fix/loadHtml

Change visibility of Readability::loadHtml() and prevent DOM reset in init()

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "composer"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/continuous-integration.yml

@@ -25,6 +25,7 @@ jobs:
           - "8.2"
           - "8.3"
           - "8.4"
+          - "8.5"
 
     steps:
       - name: "Checkout"

.php-cs-fixer.php

@@ -30,6 +30,8 @@ return (new PhpCsFixer\Config())
         'modernize_strpos' => false,
         // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
         'no_null_property_initialization' => false,
+        // @Symfony:risky started removing them but we will probably want to go the opposite direction.
+        'declare_strict_types' => false,
     ])
     ->setFinder($finder)
 ;

composer.json

@@ -30,12 +30,12 @@
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
-        "friendsofphp/php-cs-fixer": "^3.0",
+        "friendsofphp/php-cs-fixer": "3.95.12",
         "monolog/monolog": "^1.24|^2.1",
         "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0",
-        "phpstan/phpstan": "^2.0",
+        "phpstan/phpstan": "2.2.5",
-        "phpstan/phpstan-phpunit": "^2.0",
+        "phpstan/phpstan-phpunit": "2.0.18",
-        "rector/rector": "^2.0.0"
+        "rector/rector": "2.5.5"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -46,6 +46,9 @@
     "autoload-dev": {
         "psr-4": { "Tests\\Readability\\": "tests/" }
     },
+    "config": {
+        "lock": false
+    },
     "scripts": {
         "fix": "php-cs-fixer fix --verbose --diff",
         "phpstan": "phpstan analyze --memory-limit 512M",

src/JSLikeHTMLElement.php

@@ -43,7 +43,7 @@ class JSLikeHTMLElement extends \DOMElement
      * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
      * ```
      */
-    public function __set($name, $value)
+    public function __set(string $name, string $value): void
     {
         if ('innerHTML' !== $name) {
             $trace = debug_backtrace();
@@ -108,7 +108,7 @@ class JSLikeHTMLElement extends \DOMElement
      * $string = $div->innerHTML;
      * ```
      */
-    public function __get($name)
+    public function __get(string $name): string
     {
         if ('innerHTML' === $name) {
             $inner = '';
@@ -124,20 +124,22 @@ class JSLikeHTMLElement extends \DOMElement
 
         $trace = debug_backtrace();
         trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);
+
+        return '';
     }
 
-    public function __toString()
+    public function __toString(): string
     {
         return '[' . $this->tagName . ']';
     }
 
-    public function getInnerHtml()
+    public function getInnerHtml(): string
     {
         return $this->__get('innerHTML');
     }
 
-    public function setInnerHtml($value)
+    public function setInnerHtml(string $value): void
     {
-        return $this->__set('innerHTML', $value);
+        $this->__set('innerHTML', $value);
     }
 }

src/Readability.php

@@ -88,7 +88,7 @@ class Readability implements LoggerAwareInterface
         'enclose-text' => true,
         'merge-divs' => true,
         // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
         'output-encoding' => 'utf8',
         'hide-comments' => true,
     ];
@@ -213,7 +213,9 @@ class Readability implements LoggerAwareInterface
      */
     public function init(): bool
     {
+        if (!isset($this->dom)) {
             $this->loadHtml();
+        }
 
         if (!isset($this->dom->documentElement)) {
             return false;
@@ -751,6 +753,76 @@ class Readability implements LoggerAwareInterface
         $this->flags &= ~$flag;
     }
 
+    /**
+     * Load HTML in a DOMDocument.
+     * Apply Pre filters
+     * Cleanup HTML using Tidy (or not).
+     */
+    public function loadHtml(): void
+    {
+        $this->original_html = $this->html;
+
+        $this->logger->debug('Parsing URL: ' . $this->url);
+
+        if ($this->url) {
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
+        }
+
+        mb_internal_encoding('UTF-8');
+        mb_http_output('UTF-8');
+        mb_regex_encoding('UTF-8');
+
+        // HACK: dirty cleanup to replace some stuff; shouldn't use regexps with HTML but well...
+        if (!$this->flagIsActive(self::FLAG_DISABLE_PREFILTER)) {
+            foreach ($this->pre_filters as $search => $replace) {
+                $this->html = preg_replace($search, $replace, $this->html);
+            }
+            unset($search, $replace);
+        }
+
+        if ('' === trim($this->html)) {
+            $this->html = '<html></html>';
+        }
+
+        /*
+         * Use tidy (if it exists).
+         * This fixes problems with some sites which would otherwise trouble DOMDocument's HTML parsing.
+         * Although sometimes it makes matters worse, which is why there is an option to disable it.
+         */
+        if ($this->useTidy) {
+            $this->logger->debug('Tidying document');
+
+            $tidy = tidy_repair_string($this->html, $this->tidy_config, 'UTF8');
+            if (false !== $tidy && $this->html !== $tidy) {
+                $this->tidied = true;
+                $this->html = $tidy;
+                $this->html = preg_replace('/[\r\n]+/is', "\n", $this->html);
+            }
+            unset($tidy);
+        }
+
+        $this->html = self::entitizeNonAscii((string) $this->html);
+
+        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
+            $this->dom = (new HTML5())->loadHTML($this->html);
+        }
+
+        if ('libxml' === $this->parser) {
+            libxml_use_internal_errors(true);
+
+            $this->dom = new \DOMDocument();
+            $this->dom->preserveWhiteSpace = false;
+            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
+
+            libxml_use_internal_errors(false);
+        }
+
+        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
+    }
+
     /**
      * Get the article title as an H1.
      *
@@ -1003,7 +1075,11 @@ class Readability implements LoggerAwareInterface
                     }
 
                     if ($this->hasSingleTagInsideElement($node, 'p') && $this->getLinkDensity($node) < 0.25) {
-                        $newNode = $node->childNodes->item(0);
+                        // In some cases when tidy is disabled the first item may not be a DOMElement so we apply a filter
+                        $newNode = array_values(array_filter(
+                            iterator_to_array($node->childNodes),
+                            static fn ($childNode) => $childNode instanceof \DOMElement
+                        ))[0];
                         $node->parentNode->replaceChild($newNode, $node);
                         $nodesToScore[] = $newNode;
                     }
@@ -1118,7 +1194,7 @@ class Readability implements LoggerAwareInterface
 
         $topCandidates = array_filter(
             $topCandidates,
-            fn ($v, $idx) => 0 === $idx || null !== $v,
+            static fn ($v, $idx) => 0 === $idx || null !== $v,
             \ARRAY_FILTER_USE_BOTH
         );
         $topCandidate = $topCandidates[0];
@@ -1371,76 +1447,6 @@ class Readability implements LoggerAwareInterface
         }
     }
 
-    /**
-     * Load HTML in a DOMDocument.
-     * Apply Pre filters
-     * Cleanup HTML using Tidy (or not).
-     */
-    private function loadHtml(): void
-    {
-        $this->original_html = $this->html;
-
-        $this->logger->debug('Parsing URL: ' . $this->url);
-
-        if ($this->url) {
-            $host = parse_url($this->url, \PHP_URL_HOST);
-            if (null !== $host) {
-                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
-            }
-        }
-
-        mb_internal_encoding('UTF-8');
-        mb_http_output('UTF-8');
-        mb_regex_encoding('UTF-8');
-
-        // HACK: dirty cleanup to replace some stuff; shouldn't use regexps with HTML but well...
-        if (!$this->flagIsActive(self::FLAG_DISABLE_PREFILTER)) {
-            foreach ($this->pre_filters as $search => $replace) {
-                $this->html = preg_replace($search, $replace, $this->html);
-            }
-            unset($search, $replace);
-        }
-
-        if ('' === trim($this->html)) {
-            $this->html = '<html></html>';
-        }
-
-        /*
-         * Use tidy (if it exists).
-         * This fixes problems with some sites which would otherwise trouble DOMDocument's HTML parsing.
-         * Although sometimes it makes matters worse, which is why there is an option to disable it.
-         */
-        if ($this->useTidy) {
-            $this->logger->debug('Tidying document');
-
-            $tidy = tidy_repair_string($this->html, $this->tidy_config, 'UTF8');
-            if (false !== $tidy && $this->html !== $tidy) {
-                $this->tidied = true;
-                $this->html = $tidy;
-                $this->html = preg_replace('/[\r\n]+/is', "\n", $this->html);
-            }
-            unset($tidy);
-        }
-
-        $this->html = self::entitizeNonAscii((string) $this->html);
-
-        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
-            $this->dom = (new HTML5())->loadHTML($this->html);
-        }
-
-        if ('libxml' === $this->parser) {
-            libxml_use_internal_errors(true);
-
-            $this->dom = new \DOMDocument();
-            $this->dom->preserveWhiteSpace = false;
-            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
-
-            libxml_use_internal_errors(false);
-        }
-
-        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
-    }
-
     private function getAncestors(\DOMElement $node, int $maxDepth = 0): array
     {
         $ancestors = [];
@@ -1481,10 +1487,17 @@ class Readability implements LoggerAwareInterface
     private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
     {
         $childNodes = iterator_to_array($node->childNodes);
-        $children = array_filter($childNodes, fn ($childNode) => $childNode instanceof \DOMElement);
+        $children = array_filter($childNodes, static fn ($childNode) => $childNode instanceof \DOMElement);
+
+        // array_first() has been added in PHP 8.5, failing back for older versions
+        if (!\function_exists('array_first')) {
+            $firstChild = $children ? $children[array_key_first($children)] : null;
+        } else {
+            $firstChild = array_first($children);
+        }
 
         // There should be exactly 1 element child with given tag
-        if (1 !== \count($children) || $children[0]->nodeName !== $tag) {
+        if (1 !== \count($children) || $firstChild->nodeName !== $tag) {
             return false;
         }
 

tests/ReadabilityTest.php

@@ -21,6 +21,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testConstructDefault(): void
     {
         $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);
         $readability->init();
 
         $this->assertNull($readability->url);
@@ -323,7 +324,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $oldErrorReporting = error_reporting(\E_ALL);
         $oldDisplayErrors = ini_set('display_errors', '1');
         // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
+        set_error_handler(static function (int $errno, string $errstr, string $errfile, int $errline) {
             throw new \Exception($errstr, $errno);
         });
 
@@ -463,6 +464,62 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
     }
 
+    public function dataForHasSingleTagInsideElement(): array
+    {
+        return [
+            'single matching tag, no other content' => [
+                '<div><p>Some text</p></div>', 'p', true,
+            ],
+            'single matching tag with comment' => [
+                '<div><!-- comment --><p>Some text</p></div>', 'p', true,
+            ],
+            'whitespace-only text around single matching tag' => [
+                '<div>   <p>Some text</p>   </div>', 'p', true,
+            ],
+            'two matching tags' => [
+                '<div><p>One</p><p>Two</p></div>', 'p', false,
+            ],
+            'non-whitespace text alongside the single tag' => [
+                '<div>Some text<p>One</p></div>', 'p', false,
+            ],
+            'single tag with a different name' => [
+                '<div><span>One</span></div>', 'p', false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHasSingleTagInsideElement
+     */
+    public function testHasSingleTagInsideElement(string $html, string $tag, bool $expected): void
+    {
+        $dom = new \DOMDocument();
+        $dom->loadHTML('<html><body>' . $html . '</body></html>');
+        $node = $dom->getElementsByTagName('div')->item(0);
+
+        $readability = $this->getReadability('', '', 'libxml', false);
+
+        $method = new \ReflectionMethod($readability, 'hasSingleTagInsideElement');
+        if (\PHP_VERSION_ID < 80100) {
+            $method->setAccessible(true);
+        }
+
+        $this->assertSame($expected, $method->invoke($readability, $node, $tag));
+    }
+
+    public function testDivToPElementWithComment(): void
+    {
+        $text = str_repeat('Padded real article text to reach decent length for content scoring threshold here. ', 8);
+        $html = '<div><!-- comment --><code>some code snippet</code> ' . $text . '</div>';
+
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertStringContainsString('some code snippet', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('Padded real article text', $readability->getContent()->getInnerHtml());
+    }
+
     public function testKeepFootnotes(): void
     {
         // from https://www.schreibdichte.de/blog/feed-aggregator-und-spaeter-lesen-dienst-im-team
@@ -537,8 +594,8 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
-        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
         $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
     }
@@ -583,7 +640,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
         $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
-        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
         $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
     }