Merge pull request #104 from jtojnar/html-shadowing

Fix discarding `html[lang]`

.github/workflows/coding-standards.yml

@@ -11,16 +11,16 @@ on:
 jobs:
   coding-standards:
     name: "CS Fixer & PHPStan"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.2"
+          - "7.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
 
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
@@ -33,7 +33,7 @@ jobs:
           COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 

.github/workflows/continuous-integration.yml

@@ -14,22 +14,21 @@ env:
 jobs:
   phpunit:
     name: "PHPUnit (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.2"
-          - "7.3"
           - "7.4"
           - "8.0"
           - "8.1"
           - "8.2"
           - "8.3"
+          - "8.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -48,7 +47,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -60,16 +59,16 @@ jobs:
 
   phpunit-coverage:
     name: "PHPUnit coverage (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.4"
+          - "8.0"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -88,7 +87,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -99,7 +98,7 @@ jobs:
         run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
 
       - name: "Retrieve Coveralls phar"
-        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.7.0/php-coveralls.phar"
 
       - name: "Enable Coveralls phar"
         run: "chmod +x php-coveralls.phar"
@@ -111,16 +110,16 @@ jobs:
 
   phpunit-lowest:
     name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.2"
+          - "7.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -139,7 +138,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
           dependency-versions: "lowest"

.gitignore

@@ -4,3 +4,4 @@ composer.lock
 .php_cs.cache
 .php-cs-fixer.cache
 .phpunit.result.cache
+phpstan.neon

.php-cs-fixer.php

@@ -26,6 +26,10 @@ return (new PhpCsFixer\Config())
         'strict_comparison' => true,
         'strict_param' => true,
         'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony:risky but we still support PHP 7.4
+        'modernize_strpos' => false,
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
     ])
     ->setFinder($finder)
 ;

composer.json

@@ -24,7 +24,7 @@
         "role": "Developer (original JS version)"
     }],
     "require": {
-        "php": ">=7.2.0",
+        "php": ">=7.4.0",
         "ext-mbstring": "*",
         "psr/log": "^1.0.1 || ^2.0 || ^3.0",
         "masterminds/html5": "^2.7"
@@ -32,10 +32,10 @@
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.0",
         "monolog/monolog": "^1.24|^2.1",
-        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0",
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0",
-        "phpstan/phpstan": "^1.3",
+        "phpstan/phpstan": "^2.0",
-        "phpstan/phpstan-phpunit": "^1.0",
+        "phpstan/phpstan-phpunit": "^2.0",
-        "rector/rector": "^0.15.0"
+        "rector/rector": "^2.0.0"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."

phpstan.dist.neon

@@ -8,8 +8,6 @@ parameters:
     bootstrapFiles:
         - vendor/bin/.phpunit/phpunit/vendor/autoload.php
 
-    checkMissingIterableValueType: false
-
 includes:
     - vendor/phpstan/phpstan-phpunit/extension.neon
     - vendor/phpstan/phpstan-phpunit/rules.neon

rector.php

@@ -3,29 +3,16 @@
 declare(strict_types=1);
 
 use Rector\Config\RectorConfig;
-use Rector\Core\ValueObject\PhpVersion;
 use Rector\Set\ValueObject\LevelSetList;
 
-return static function (RectorConfig $rectorConfig): void {
+return RectorConfig::configure()
-    // paths to refactor; solid alternative to CLI arguments
+    ->withPaths([
-    $rectorConfig->paths([
         __DIR__ . '/src',
         __DIR__ . '/tests',
-    ]);
+    ])
-
+    ->withBootstrapFiles([
-    // Path to phpstan with extensions, that PHPSTan in Rector uses to determine types
-    $rectorConfig->phpstanConfig(__DIR__ . '/phpstan.neon');
-
-    $rectorConfig->bootstrapFiles([
         __DIR__ . '/vendor/bin/.phpunit/phpunit/vendor/autoload.php',
         __DIR__ . '/vendor/autoload.php',
-    ]);
+    ])
-
+    ->withSets([LevelSetList::UP_TO_PHP_74])
-    // Define what rule sets will be applied
+;
-    $rectorConfig->sets([
-        LevelSetList::UP_TO_PHP_72,
-    ]);
-
-    // is your PHP version different from the one your refactor to?
-    $rectorConfig->phpVersion(PhpVersion::PHP_72);
-};

src/Readability.php

@@ -142,7 +142,7 @@ class Readability implements LoggerAwareInterface
      * @param string $parser  Which parser to use for turning raw HTML into a DOMDocument
      * @param bool   $useTidy Use tidy
      */
-    public function __construct(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true)
+    public function __construct(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true)
     {
         $this->url = $url;
         $this->html = $html;
@@ -302,8 +302,7 @@ class Readability implements LoggerAwareInterface
         $articleLinks = $articleContent->getElementsByTagName('a');
         $linkCount = 0;
 
-        for ($i = 0; $i < $articleLinks->length; ++$i) {
+        foreach ($articleLinks as $articleLink) {
-            $articleLink = $articleLinks->item($i);
             $footnoteLink = $articleLink->cloneNode(true);
             $refLink = $this->dom->createElement('a');
             $footnote = $this->dom->createElement('li');
@@ -383,8 +382,8 @@ class Readability implements LoggerAwareInterface
 
         // Remove service data-candidate attribute.
         $elems = $xpath->query('.//*[@data-candidate]', $articleContent);
-        for ($i = $elems->length - 1; $i >= 0; --$i) {
+        foreach ($elems as $elem) {
-            $elems->item($i)->removeAttribute('data-candidate');
+            $elem->removeAttribute('data-candidate');
         }
 
         // Clean out junk from the article content.
@@ -520,11 +519,12 @@ class Readability implements LoggerAwareInterface
         $textLength = mb_strlen($this->getInnerText($e, true, true));
         $linkLength = 0;
 
-        for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; ++$i) {
+        $dRe = $this->domainRegExp;
-            if ($excludeExternal && $dRe && !preg_match($dRe, $links->item($i)->getAttribute('href'))) {
+        foreach ($links as $link) {
+            if ($excludeExternal && $dRe && !preg_match($dRe, $link->getAttribute('href'))) {
                 continue;
             }
-            $linkLength += mb_strlen($this->getInnerText($links->item($i)));
+            $linkLength += mb_strlen($this->getInnerText($link));
         }
 
         if ($textLength > 0 && $linkLength > 0) {
@@ -586,7 +586,7 @@ class Readability implements LoggerAwareInterface
                 }
 
                 // Then check the elements inside this element for the same.
-                if (preg_match($this->regexps['media'], $targetList->item($y)->getInnerHTML())) {
+                if (preg_match($this->regexps['media'], $currentItem->getInnerHTML())) {
                     continue;
                 }
             }
@@ -640,15 +640,15 @@ class Readability implements LoggerAwareInterface
                 $embedCount = 0;
                 $embeds = $node->getElementsByTagName('embed');
 
-                for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
+                foreach ($embeds as $embed) {
-                    if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
+                    if (preg_match($this->regexps['media'], $embed->getAttribute('src'))) {
                         ++$embedCount;
                     }
                 }
 
                 $embeds = $node->getElementsByTagName('iframe');
-                for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
+                foreach ($embeds as $embed) {
-                    if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
+                    if (preg_match($this->regexps['media'], $embed->getAttribute('src'))) {
                         ++$embedCount;
                     }
                 }
@@ -668,10 +668,10 @@ class Readability implements LoggerAwareInterface
                         $this->logger->debug(' content length less than 6 chars, 0 embeds and either 0 images or more than 2 images');
                         $toRemove = true;
                     } elseif (!$isList && $weight < 25 && $linkDensity > 0.25) {
-                        $this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.25');
+                        $this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.25');
                         $toRemove = true;
                     } elseif ($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
-                        $this->logger->debug('  more than 2 links and weight is ' . $weight . ' > 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
+                        $this->logger->debug('  more than 2 links and weight is ' . $weight . ' > 25 but link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.5');
                         $toRemove = true;
                     } elseif ($embedCount > 3) {
                         $this->logger->debug(' more than 3 embeds');
@@ -691,10 +691,10 @@ class Readability implements LoggerAwareInterface
                         $this->logger->debug('  content length less than 10 chars and 0 images, or more than 2 images');
                         $toRemove = true;
                     } elseif (!$isList && $weight < 25 && $linkDensity > 0.2) {
-                        $this->logger->debug('  weight is ' . $weight . ' lower than 0 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.2');
+                        $this->logger->debug('  weight is ' . $weight . ' lower than 0 and link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.2');
                         $toRemove = true;
                     } elseif ($weight >= 25 && $linkDensity > 0.5) {
-                        $this->logger->debug('  weight above 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
+                        $this->logger->debug('  weight above 25 but link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.5');
                         $toRemove = true;
                     } elseif ((1 === $embedCount && $contentLength < 75) || $embedCount > 1) {
                         $this->logger->debug('  1 embed and content length smaller than 75 chars, or more than one embed');
@@ -719,8 +719,9 @@ class Readability implements LoggerAwareInterface
             $headers = $e->getElementsByTagName('h' . $headerIndex);
 
             for ($i = $headers->length - 1; $i >= 0; --$i) {
-                if ($this->getWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
+                $header = $headers->item($i);
-                    $headers->item($i)->parentNode->removeChild($headers->item($i));
+                if ($this->getWeight($header) < 0 || $this->getLinkDensity($header) > 0.33) {
+                    $header->parentNode->removeChild($header);
                 }
             }
         }
@@ -739,7 +740,7 @@ class Readability implements LoggerAwareInterface
      */
     public function addFlag(int $flag): void
     {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
     }
 
     /**
@@ -747,7 +748,7 @@ class Readability implements LoggerAwareInterface
      */
     public function removeFlag(int $flag): void
     {
-        $this->flags = $this->flags & ~$flag;
+        $this->flags &= ~$flag;
     }
 
     /**
@@ -812,12 +813,14 @@ class Readability implements LoggerAwareInterface
         // Remove all style tags in head.
         $styleTags = $this->dom->getElementsByTagName('style');
         for ($i = $styleTags->length - 1; $i >= 0; --$i) {
-            $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
+            $styleTag = $styleTags->item($i);
+            $styleTag->parentNode->removeChild($styleTag);
         }
 
         $linkTags = $this->dom->getElementsByTagName('link');
         for ($i = $linkTags->length - 1; $i >= 0; --$i) {
-            $linkTags->item($i)->parentNode->removeChild($linkTags->item($i));
+            $linkTag = $linkTags->item($i);
+            $linkTag->parentNode->removeChild($linkTag);
         }
     }
 
@@ -893,11 +896,9 @@ class Readability implements LoggerAwareInterface
      * Using a variety of metrics (content score, classname, element types), find the content that is
      * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
      *
-     * @param \DOMElement $page
-     *
      * @return \DOMElement|false
      */
-    protected function grabArticle(\DOMElement $page = null)
+    protected function grabArticle(?\DOMElement $page = null)
     {
         if (!$page) {
             $page = $this->dom;
@@ -933,9 +934,9 @@ class Readability implements LoggerAwareInterface
             // Remove unlikely candidates
             $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
 
-            if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
+            if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
-                preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+                && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
-                !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+                && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
             ) {
                 $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '"');
                 $node->parentNode->removeChild($node);
@@ -1017,15 +1018,15 @@ class Readability implements LoggerAwareInterface
          * A score is determined by things like number of commas, class names, etc.
          * Maybe eventually link density.
          */
-        for ($pt = 0, $scored = \count($nodesToScore); $pt < $scored; ++$pt) {
+        foreach ($nodesToScore as $nodeToScore) {
-            $ancestors = $this->getAncestors($nodesToScore[$pt], 5);
+            $ancestors = $this->getAncestors($nodeToScore, 5);
 
             // No parent node? Move on...
             if (0 === \count($ancestors)) {
                 continue;
             }
 
-            $innerText = $this->getInnerText($nodesToScore[$pt]);
+            $innerText = $this->getInnerText($nodeToScore);
 
             // If this paragraph is less than MIN_PARAGRAPH_LENGTH (default:20) characters, don't even count it.
             if (mb_strlen($innerText) < self::MIN_PARAGRAPH_LENGTH) {
@@ -1078,11 +1079,6 @@ class Readability implements LoggerAwareInterface
                 }
             }
 
-            $candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
-
-            for ($c = $candidates->length - 1; $c >= 0; --$c) {
-                $node = $candidates->item($c);
-            }
             unset($candidates);
         }
 
@@ -1120,9 +1116,11 @@ class Readability implements LoggerAwareInterface
             }
         }
 
-        $topCandidates = array_filter($topCandidates, function ($v, $idx) {
+        $topCandidates = array_filter(
-            return 0 === $idx || null !== $v;
+            $topCandidates,
-        }, \ARRAY_FILTER_USE_BOTH);
+            fn ($v, $idx) => 0 === $idx || null !== $v,
+            \ARRAY_FILTER_USE_BOTH
+        );
         $topCandidate = $topCandidates[0];
 
         /*
@@ -1231,11 +1229,6 @@ class Readability implements LoggerAwareInterface
         $parentOfTopCandidate = $topCandidate->parentNode;
         $siblingNodes = $parentOfTopCandidate->childNodes;
 
-        if (0 === $siblingNodes->length) {
-            $siblingNodes = new \stdClass();
-            $siblingNodes->length = 0;
-        }
-
         for ($s = 0, $sl = $siblingNodes->length; $s < $sl; ++$s) {
             $siblingNode = $siblingNodes->item($s);
             $siblingNodeName = $siblingNode->nodeName;
@@ -1390,7 +1383,10 @@ class Readability implements LoggerAwareInterface
         $this->logger->debug('Parsing URL: ' . $this->url);
 
         if ($this->url) {
-            $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', (string) parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
         }
 
         mb_internal_encoding('UTF-8');
@@ -1426,7 +1422,7 @@ class Readability implements LoggerAwareInterface
             unset($tidy);
         }
 
-        $this->html = '<meta charset="utf-8">' . (string) $this->html;
+        $this->html = self::ensureMetaCharset((string) $this->html);
 
         if ('html5lib' === $this->parser || 'html5' === $this->parser) {
             $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1442,7 +1438,7 @@ class Readability implements LoggerAwareInterface
             libxml_use_internal_errors(false);
         }
 
-        $this->dom->registerNodeClass(\DOMElement::class, \Readability\JSLikeHTMLElement::class);
+        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
     }
 
     private function getAncestors(\DOMElement $node, int $maxDepth = 0): array
@@ -1464,21 +1460,38 @@ class Readability implements LoggerAwareInterface
     {
         return \XML_TEXT_NODE === $node->nodeType
             || \in_array(strtoupper($node->nodeName), $this->phrasingElements, true)
-            || (\in_array(strtoupper($node->nodeName), ['A', 'DEL', 'INS'], true) && !\in_array(false, array_map(function ($c) {
+            || (
-                return $this->isPhrasingContent($c);
+                \in_array(strtoupper($node->nodeName), ['A', 'DEL', 'INS'], true)
-            }, iterator_to_array($node->childNodes)), true));
+                && !\in_array(
+                    false,
+                    array_map(
+                        fn ($c) => $this->isPhrasingContent($c),
+                        iterator_to_array($node->childNodes)
+                    ),
+                    true
+                )
+            );
     }
 
+    /**
+     * Checks if `$node` has only whitespace and a single element with `$tag` for the tag name.
+     * Returns false if `$node` contains non-empty text nodes
+     * or if it contains no element with given tag or more than 1 element.
+     */
     private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
     {
-        if (1 !== $node->childNodes->length || $node->childNodes->item(0)->nodeName !== $tag) {
+        $childNodes = iterator_to_array($node->childNodes);
+        $children = array_filter($childNodes, fn ($childNode) => $childNode instanceof \DOMElement);
+
+        // There should be exactly 1 element child with given tag
+        if (1 !== \count($children) || $children[0]->nodeName !== $tag) {
             return false;
         }
 
-        $a = array_filter(iterator_to_array($node->childNodes), function ($childNode) {
+        $a = array_filter(
-            return $childNode instanceof \DOMText &&
+            $childNodes,
-                preg_match($this->regexps['hasContent'], $this->getInnerText($childNode));
+            fn ($childNode) => $childNode instanceof \DOMText && preg_match($this->regexps['hasContent'], $this->getInnerText($childNode))
-        });
+        );
 
         return 0 === \count($a);
     }
@@ -1491,9 +1504,51 @@ class Readability implements LoggerAwareInterface
      */
     private function isNodeVisible(\DOMElement $node): bool
     {
-        return !($node->hasAttribute('style')
+        return !(
-                    && preg_match($this->regexps['isNotVisible'], $node->getAttribute('style'))
+            $node->hasAttribute('style')
+            && preg_match($this->regexps['isNotVisible'], $node->getAttribute('style'))
         )
-                && !$node->hasAttribute('hidden');
+        && !$node->hasAttribute('hidden');
+    }
+
+    /**
+     * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document.
+     *
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag.
+     *
+     * @param string $html UTF-8 encoded document
+     */
+    private static function ensureMetaCharset(string $html): string
+    {
+        $charsetTag = '<meta charset="utf-8">';
+
+        // Only look at first 1024 bytes since, according to HTML5 specification,
+        // that’s where <meta> elements declaring a character encoding must be located.
+        // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset
+        $start = substr($html, 0, 1000);
+
+        if (1 === preg_match('/<meta[^>]+charset/i', $start)) {
+            // <meta> tag is already present, no need for modification.
+            return $html;
+        }
+
+        if (1 === preg_match('/<head[^>]*>/i', $start)) {
+            // <head> tag was located, <meta> tags go there.
+            $html = preg_replace('/<head[^>]*>/i', '$0' . $charsetTag, $html, 1);
+
+            return $html;
+        }
+
+        if (1 === preg_match('/<html[^>]*>/i', $start)) {
+            // <html> tag was located, let’s put it inside and have parser create <head>.
+            $html = preg_replace('/<html[^>]*>/i', '$0' . $charsetTag, $html, 1);
+
+            return $html;
+        }
+
+        // Fallback – just plop the <meta> at the start of the fragment.
+        return $charsetTag . $html;
     }
 }

tests/ReadabilityTest.php

@@ -24,7 +24,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructHtml5Parser(): void
@@ -33,7 +33,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
     }
 
@@ -46,7 +46,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertTrue($readability->tidied);
     }
@@ -60,7 +60,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertSame('', $readability->original_html);
         $this->assertFalse($readability->tidied);
 
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructSimpleWithoutTidy(): void
@@ -69,7 +69,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertFalse($readability->tidied);
     }
@@ -115,7 +115,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testInitDiv(): void
     {
         $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -129,7 +128,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithFootnotes(): void
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -146,7 +144,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testStandardClean(): void
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -163,7 +160,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithIframe(): void
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -178,7 +174,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithArticle(): void
     {
         $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -193,7 +188,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithAside(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -208,7 +202,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClasses(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -223,7 +216,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClassesWithoutLightClean(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -239,7 +231,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithTd(): void
     {
         $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -252,7 +243,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithSameClasses(): void
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -266,7 +256,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithScript(): void
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -280,7 +269,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitle(): void
     {
         $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -294,7 +282,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDash(): void
     {
         $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -308,7 +295,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDoubleDot(): void
     {
         $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -322,7 +308,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleTooShortUseH1(): void
     {
         $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -335,12 +320,12 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
 
     public function testAutoClosingIframeNotThrowingException(): void
     {
-        $oldErrorReporting = error_reporting(\E_ALL | \E_STRICT);
+        $oldErrorReporting = error_reporting(\E_ALL);
         $oldDisplayErrors = ini_set('display_errors', '1');
         // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline, array $errcontext) {
+        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
             throw new \Exception($errstr, $errno);
-        }, \E_ALL | \E_STRICT);
+        });
 
         try {
             $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -369,7 +354,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
                 </html>';
 
             $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-            $readability->debug = true;
 
             $res = $readability->init();
 
@@ -437,7 +421,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             </html>';
 
         $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;
 
         $res = $readability->init();
 
@@ -474,7 +457,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/childNodeGoesNull.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -487,7 +469,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/keepFootnotes.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -501,7 +482,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/wipedBody.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -540,7 +520,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testVisibleNode(string $content, bool $shouldBeVisible): void
     {
         $readability = $this->getReadability($content, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         if ($shouldBeVisible) {
@@ -550,7 +529,49 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         }
     }
 
-    private function getReadability(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang(): array
+    {
+        return [
+            'meta' => [
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'head' => [
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
+                false,
+            ],
+            'fragment' => [
+                '<article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article>',
+                '',
+                // tidy would add <html>.
+                false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHtmlLang
+     */
+    public function testHtmlLang(string $html, string $lang, bool $useTidy = true): void
+    {
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+    }
+
+    private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
     {
         $readability = new Readability($html, $url, $parser, $useTidy);