Merge pull request #106 from jtojnar/encode

Fix character decoding regression when `title` precedes `meta[charset]`

.github/workflows/coding-standards.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   coding-standards:
     name: "CS Fixer & PHPStan"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
 
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
@@ -33,7 +33,7 @@ jobs:
           COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 

.github/workflows/continuous-integration.yml

@@ -14,20 +14,21 @@ env:
 jobs:
   phpunit:
     name: "PHPUnit (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.2"
-          - "7.3"
           - "7.4"
           - "8.0"
           - "8.1"
+          - "8.2"
+          - "8.3"
+          - "8.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -46,7 +47,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -58,16 +59,16 @@ jobs:
 
   phpunit-coverage:
     name: "PHPUnit coverage (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.4"
+          - "8.0"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -86,7 +87,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -97,7 +98,7 @@ jobs:
         run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
 
       - name: "Retrieve Coveralls phar"
-        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.7.0/php-coveralls.phar"
 
       - name: "Enable Coveralls phar"
         run: "chmod +x php-coveralls.phar"
@@ -109,16 +110,16 @@ jobs:
 
   phpunit-lowest:
     name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
 
     strategy:
       matrix:
         php:
-          - "7.2"
+          - "7.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -137,7 +138,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
           dependency-versions: "lowest"

.gitignore

@@ -4,3 +4,4 @@ composer.lock
 .php_cs.cache
 .php-cs-fixer.cache
 .phpunit.result.cache
+phpstan.neon

.php-cs-fixer.php

@@ -26,6 +26,10 @@ return (new PhpCsFixer\Config())
         'strict_comparison' => true,
         'strict_param' => true,
         'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony:risky but we still support PHP 7.4
+        'modernize_strpos' => false,
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
     ])
     ->setFinder($finder)
 ;

composer.json

@@ -24,18 +24,18 @@
         "role": "Developer (original JS version)"
     }],
     "require": {
-        "php": ">=7.2.0",
+        "php": ">=7.4.0",
         "ext-mbstring": "*",
-        "psr/log": "^1.0",
+        "psr/log": "^1.0.1 || ^2.0 || ^3.0",
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.0",
         "monolog/monolog": "^1.24|^2.1",
-        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0",
-        "phpstan/phpstan": "^1.3",
-        "phpstan/phpstan-phpunit": "^1.0",
-        "rector/rector": "^0.12.15"
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0",
+        "phpstan/phpstan": "^2.0",
+        "phpstan/phpstan-phpunit": "^2.0",
+        "rector/rector": "^2.0.0"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -46,9 +46,10 @@
     "autoload-dev": {
         "psr-4": { "Tests\\Readability\\": "tests/" }
     },
-    "config":{
-        "platform": {
-            "php": "7.2.34"
-        }
+    "scripts": {
+        "fix": "php-cs-fixer fix --verbose --diff",
+        "phpstan": "phpstan analyze --memory-limit 512M",
+        "rector": "rector process",
+        "test": "simple-phpunit -v"
     }
 }

phpstan.dist.neon

@@ -6,9 +6,7 @@ parameters:
 
     # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
     bootstrapFiles:
-        - vendor/bin/.phpunit/phpunit-8.5-0/vendor/autoload.php
-
-    checkMissingIterableValueType: false
+        - vendor/bin/.phpunit/phpunit/vendor/autoload.php
 
 includes:
     - vendor/phpstan/phpstan-phpunit/extension.neon

rector.php

@@ -2,31 +2,17 @@
 
 declare(strict_types=1);
 
-use Rector\Core\Configuration\Option;
-use Rector\Core\ValueObject\PhpVersion;
+use Rector\Config\RectorConfig;
 use Rector\Set\ValueObject\LevelSetList;
-use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator;
 
-return static function (ContainerConfigurator $containerConfigurator): void {
-    $parameters = $containerConfigurator->parameters();
-
-    // paths to refactor; solid alternative to CLI arguments
-    $parameters->set(Option::PATHS, [
+return RectorConfig::configure()
+    ->withPaths([
         __DIR__ . '/src',
         __DIR__ . '/tests',
-    ]);
-
-    // Path to phpstan with extensions, that PHPSTan in Rector uses to determine types
-    $parameters->set(Option::PHPSTAN_FOR_RECTOR_PATH, __DIR__ . '/phpstan.neon');
-
-    $parameters->set(Option::BOOTSTRAP_FILES, [
-        __DIR__ . '/vendor/bin/.phpunit/phpunit-8.5-0/vendor/autoload.php',
+    ])
+    ->withBootstrapFiles([
+        __DIR__ . '/vendor/bin/.phpunit/phpunit/vendor/autoload.php',
         __DIR__ . '/vendor/autoload.php',
-    ]);
-
-    // Define what rule sets will be applied
-    $containerConfigurator->import(LevelSetList::UP_TO_PHP_72);
-
-    // is your PHP version different from the one your refactor to?
-    $parameters->set(Option::PHP_VERSION_FEATURES, PhpVersion::PHP_72);
-};
+    ])
+    ->withSets([LevelSetList::UP_TO_PHP_74])
+;

src/JSLikeHTMLElement.php

@@ -39,9 +39,9 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for setting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
-     * @endcode
+     * ```
      */
     public function __set($name, $value)
     {
@@ -79,14 +79,13 @@ class JSLikeHTMLElement extends \DOMElement
         } else {
             // $value is probably ill-formed
             $f = new \DOMDocument();
-            $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
 
             // Using <htmlfragment> will generate a warning, but so will bad HTML
             // (and by this point, bad HTML is what we've got).
             // We use it (and suppress the warning) because an HTML fragment will
             // be wrapped around <html><body> tags which we don't really want to keep.
             // Note: despite the warning, if loadHTML succeeds it will return true.
-            $result = $f->loadHTML('<htmlfragment>' . $value . '</htmlfragment>');
+            $result = $f->loadHTML('<meta charset="utf-8"><htmlfragment>' . $value . '</htmlfragment>');
 
             if ($result) {
                 $import = $f->getElementsByTagName('htmlfragment')->item(0);
@@ -105,9 +104,9 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for getting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $string = $div->innerHTML;
-     * @endcode
+     * ```
      */
     public function __get($name)
     {

src/Readability.php

@@ -2,7 +2,6 @@
 
 namespace Readability;
 
-use DOMElement;
 use Masterminds\HTML5;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerInterface;
@@ -115,7 +114,7 @@ class Readability implements LoggerAwareInterface
         // HACK: replace linebreaks plus br's with p's
         '!(<br[^>]*>[ \r\n\s]*){2,}!i' => '</p><p>',
         // replace noscripts
-        //'!</?noscript>!is' => '',
+        // '!</?noscript>!is' => '',
         // replace fonts to spans
         '!<(/?)font[^>]*>!is' => '<\\1span>',
     ];
@@ -126,8 +125,8 @@ class Readability implements LoggerAwareInterface
         // replace empty tags that break layouts
         '!<(?:a|div|p|figure)[^>]+/>!is' => '',
         // remove all attributes on text tags
-        //'!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
-        //single newlines cleanup
+        // '!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
+        // single newlines cleanup
         "/\n+/" => "\n",
         // modern web...
         '!<pre[^>]*>\s*<code!is' => '<pre',
@@ -143,7 +142,7 @@ class Readability implements LoggerAwareInterface
      * @param string $parser  Which parser to use for turning raw HTML into a DOMDocument
      * @param bool   $useTidy Use tidy
      */
-    public function __construct(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true)
+    public function __construct(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true)
     {
         $this->url = $url;
         $this->html = $html;
@@ -161,7 +160,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Get article title element.
      *
-     * @return DOMElement
+     * @return \DOMElement
      */
     public function getTitle()
     {
@@ -171,7 +170,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Get article content element.
      *
-     * @return DOMElement
+     * @return \DOMElement
      */
     public function getContent()
     {
@@ -280,7 +279,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Run any post-process modifications to article content as necessary.
      */
-    public function postProcessContent(DOMElement $articleContent): void
+    public function postProcessContent(\DOMElement $articleContent): void
     {
         if ($this->convertLinksToFootnotes && !preg_match('/\bwiki/', $this->url)) {
             $this->addFootnotes($articleContent);
@@ -292,7 +291,7 @@ class Readability implements LoggerAwareInterface
      *
      * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
      */
-    public function addFootnotes(DOMElement $articleContent): void
+    public function addFootnotes(\DOMElement $articleContent): void
     {
         $footnotesWrapper = $this->dom->createElement('footer');
         $footnotesWrapper->setAttribute('class', 'readability-footnotes');
@@ -303,8 +302,7 @@ class Readability implements LoggerAwareInterface
         $articleLinks = $articleContent->getElementsByTagName('a');
         $linkCount = 0;
 
-        for ($i = 0; $i < $articleLinks->length; ++$i) {
-            $articleLink = $articleLinks->item($i);
+        foreach ($articleLinks as $articleLink) {
             $footnoteLink = $articleLink->cloneNode(true);
             $refLink = $this->dom->createElement('a');
             $footnote = $this->dom->createElement('li');
@@ -335,7 +333,7 @@ class Readability implements LoggerAwareInterface
             $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
             $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
             $footnote->setInnerHtml('<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ');
-            $footnoteLink->setInnerHtml(('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText));
+            $footnoteLink->setInnerHtml('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText);
             $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
             $footnote->appendChild($footnoteLink);
 
@@ -356,7 +354,7 @@ class Readability implements LoggerAwareInterface
      */
     public function prepArticle(\DOMNode $articleContent): void
     {
-        if (!$articleContent instanceof DOMElement) {
+        if (!$articleContent instanceof \DOMElement) {
             return;
         }
 
@@ -384,8 +382,8 @@ class Readability implements LoggerAwareInterface
 
         // Remove service data-candidate attribute.
         $elems = $xpath->query('.//*[@data-candidate]', $articleContent);
-        for ($i = $elems->length - 1; $i >= 0; --$i) {
-            $elems->item($i)->removeAttribute('data-candidate');
+        foreach ($elems as $elem) {
+            $elem->removeAttribute('data-candidate');
         }
 
         // Clean out junk from the article content.
@@ -456,9 +454,9 @@ class Readability implements LoggerAwareInterface
      * Get the inner text of a node.
      * This also strips out any excess whitespace to be found.
      *
-     * @param DOMElement $e
-     * @param bool       $normalizeSpaces (default: true)
-     * @param bool       $flattenLines    (default: false)
+     * @param \DOMElement $e
+     * @param bool        $normalizeSpaces (default: true)
+     * @param bool        $flattenLines    (default: false)
      */
     public function getInnerText($e, bool $normalizeSpaces = true, bool $flattenLines = false): string
     {
@@ -482,7 +480,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Remove the style attribute on every $e and under.
      */
-    public function cleanStyles(DOMElement $e): void
+    public function cleanStyles(\DOMElement $e): void
     {
         if (\is_object($e)) {
             $elems = $e->getElementsByTagName('*');
@@ -515,17 +513,18 @@ class Readability implements LoggerAwareInterface
      * This is the amount of text that is inside a link divided by the total text in the node.
      * Can exclude external references to differentiate between simple text and menus/infoblocks.
      */
-    public function getLinkDensity(DOMElement $e, bool $excludeExternal = false): float
+    public function getLinkDensity(\DOMElement $e, bool $excludeExternal = false): float
     {
         $links = $e->getElementsByTagName('a');
         $textLength = mb_strlen($this->getInnerText($e, true, true));
         $linkLength = 0;
 
-        for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; ++$i) {
-            if ($excludeExternal && $dRe && !preg_match($dRe, $links->item($i)->getAttribute('href'))) {
+        $dRe = $this->domainRegExp;
+        foreach ($links as $link) {
+            if ($excludeExternal && $dRe && !preg_match($dRe, $link->getAttribute('href'))) {
                 continue;
             }
-            $linkLength += mb_strlen($this->getInnerText($links->item($i)));
+            $linkLength += mb_strlen($this->getInnerText($link));
         }
 
         if ($textLength > 0 && $linkLength > 0) {
@@ -538,7 +537,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Get an element relative weight.
      */
-    public function getWeight(DOMElement $e): int
+    public function getWeight(\DOMElement $e): int
     {
         if (!$this->flagIsActive(self::FLAG_WEIGHT_ATTRIBUTES)) {
             return 0;
@@ -556,7 +555,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Remove extraneous break tags from a node.
      */
-    public function killBreaks(DOMElement $node): void
+    public function killBreaks(\DOMElement $node): void
     {
         $html = $node->getInnerHTML();
         $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
@@ -569,7 +568,7 @@ class Readability implements LoggerAwareInterface
      *
      * Updated 2012-09-18 to preserve youtube/vimeo iframes
      */
-    public function clean(DOMElement $e, string $tag): void
+    public function clean(\DOMElement $e, string $tag): void
     {
         $targetList = $e->getElementsByTagName($tag);
         $isEmbed = ('audio' === $tag || 'video' === $tag || 'iframe' === $tag || 'object' === $tag || 'embed' === $tag);
@@ -587,7 +586,7 @@ class Readability implements LoggerAwareInterface
                 }
 
                 // Then check the elements inside this element for the same.
-                if (preg_match($this->regexps['media'], $targetList->item($y)->getInnerHTML())) {
+                if (preg_match($this->regexps['media'], $currentItem->getInnerHTML())) {
                     continue;
                 }
             }
@@ -601,7 +600,7 @@ class Readability implements LoggerAwareInterface
      * "Fishy" is an algorithm based on content length, classnames,
      * link density, number of images & embeds, etc.
      */
-    public function cleanConditionally(DOMElement $e, string $tag): void
+    public function cleanConditionally(\DOMElement $e, string $tag): void
     {
         if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
             return;
@@ -641,15 +640,15 @@ class Readability implements LoggerAwareInterface
                 $embedCount = 0;
                 $embeds = $node->getElementsByTagName('embed');
 
-                for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
-                    if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
+                foreach ($embeds as $embed) {
+                    if (preg_match($this->regexps['media'], $embed->getAttribute('src'))) {
                         ++$embedCount;
                     }
                 }
 
                 $embeds = $node->getElementsByTagName('iframe');
-                for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
-                    if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
+                foreach ($embeds as $embed) {
+                    if (preg_match($this->regexps['media'], $embed->getAttribute('src'))) {
                         ++$embedCount;
                     }
                 }
@@ -669,10 +668,10 @@ class Readability implements LoggerAwareInterface
                         $this->logger->debug(' content length less than 6 chars, 0 embeds and either 0 images or more than 2 images');
                         $toRemove = true;
                     } elseif (!$isList && $weight < 25 && $linkDensity > 0.25) {
-                        $this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.25');
+                        $this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.25');
                         $toRemove = true;
                     } elseif ($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
-                        $this->logger->debug('  more than 2 links and weight is ' . $weight . ' > 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
+                        $this->logger->debug('  more than 2 links and weight is ' . $weight . ' > 25 but link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.5');
                         $toRemove = true;
                     } elseif ($embedCount > 3) {
                         $this->logger->debug(' more than 3 embeds');
@@ -692,10 +691,10 @@ class Readability implements LoggerAwareInterface
                         $this->logger->debug('  content length less than 10 chars and 0 images, or more than 2 images');
                         $toRemove = true;
                     } elseif (!$isList && $weight < 25 && $linkDensity > 0.2) {
-                        $this->logger->debug('  weight is ' . $weight . ' lower than 0 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.2');
+                        $this->logger->debug('  weight is ' . $weight . ' lower than 0 and link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.2');
                         $toRemove = true;
                     } elseif ($weight >= 25 && $linkDensity > 0.5) {
-                        $this->logger->debug('  weight above 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
+                        $this->logger->debug('  weight above 25 but link density is ' . \sprintf('%.2f', $linkDensity) . ' > 0.5');
                         $toRemove = true;
                     } elseif ((1 === $embedCount && $contentLength < 75) || $embedCount > 1) {
                         $this->logger->debug('  1 embed and content length smaller than 75 chars, or more than one embed');
@@ -714,14 +713,15 @@ class Readability implements LoggerAwareInterface
     /**
      * Clean out spurious headers from an Element. Checks things like classnames and link density.
      */
-    public function cleanHeaders(DOMElement $e): void
+    public function cleanHeaders(\DOMElement $e): void
     {
         for ($headerIndex = 1; $headerIndex < 3; ++$headerIndex) {
             $headers = $e->getElementsByTagName('h' . $headerIndex);
 
             for ($i = $headers->length - 1; $i >= 0; --$i) {
-                if ($this->getWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
-                    $headers->item($i)->parentNode->removeChild($headers->item($i));
+                $header = $headers->item($i);
+                if ($this->getWeight($header) < 0 || $this->getLinkDensity($header) > 0.33) {
+                    $header->parentNode->removeChild($header);
                 }
             }
         }
@@ -740,7 +740,7 @@ class Readability implements LoggerAwareInterface
      */
     public function addFlag(int $flag): void
     {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
     }
 
     /**
@@ -748,13 +748,13 @@ class Readability implements LoggerAwareInterface
      */
     public function removeFlag(int $flag): void
     {
-        $this->flags = $this->flags & ~$flag;
+        $this->flags &= ~$flag;
     }
 
     /**
      * Get the article title as an H1.
      *
-     * @return DOMElement
+     * @return \DOMElement
      */
     protected function getArticleTitle()
     {
@@ -813,12 +813,14 @@ class Readability implements LoggerAwareInterface
         // Remove all style tags in head.
         $styleTags = $this->dom->getElementsByTagName('style');
         for ($i = $styleTags->length - 1; $i >= 0; --$i) {
-            $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
+            $styleTag = $styleTags->item($i);
+            $styleTag->parentNode->removeChild($styleTag);
         }
 
         $linkTags = $this->dom->getElementsByTagName('link');
         for ($i = $linkTags->length - 1; $i >= 0; --$i) {
-            $linkTags->item($i)->parentNode->removeChild($linkTags->item($i));
+            $linkTag = $linkTags->item($i);
+            $linkTag->parentNode->removeChild($linkTag);
         }
     }
 
@@ -826,7 +828,7 @@ class Readability implements LoggerAwareInterface
      * Initialize a node with the readability object. Also checks the
      * className/id for special names to add to its score.
      */
-    protected function initializeNode(DOMElement $node): void
+    protected function initializeNode(\DOMElement $node): void
     {
         if (!isset($node->tagName)) {
             return;
@@ -894,11 +896,9 @@ class Readability implements LoggerAwareInterface
      * Using a variety of metrics (content score, classname, element types), find the content that is
      * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
      *
-     * @param DOMElement $page
-     *
-     * @return DOMElement|false
+     * @return \DOMElement|false
      */
-    protected function grabArticle(DOMElement $page = null)
+    protected function grabArticle(?\DOMElement $page = null)
     {
         if (!$page) {
             $page = $this->dom;
@@ -934,9 +934,9 @@ class Readability implements LoggerAwareInterface
             // Remove unlikely candidates
             $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
 
-            if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
-                preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
-                !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+            if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
+                && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
+                && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
             ) {
                 $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '"');
                 $node->parentNode->removeChild($node);
@@ -1018,15 +1018,15 @@ class Readability implements LoggerAwareInterface
          * A score is determined by things like number of commas, class names, etc.
          * Maybe eventually link density.
          */
-        for ($pt = 0, $scored = \count($nodesToScore); $pt < $scored; ++$pt) {
-            $ancestors = $this->getAncestors($nodesToScore[$pt], 5);
+        foreach ($nodesToScore as $nodeToScore) {
+            $ancestors = $this->getAncestors($nodeToScore, 5);
 
             // No parent node? Move on...
             if (0 === \count($ancestors)) {
                 continue;
             }
 
-            $innerText = $this->getInnerText($nodesToScore[$pt]);
+            $innerText = $this->getInnerText($nodeToScore);
 
             // If this paragraph is less than MIN_PARAGRAPH_LENGTH (default:20) characters, don't even count it.
             if (mb_strlen($innerText) < self::MIN_PARAGRAPH_LENGTH) {
@@ -1040,7 +1040,7 @@ class Readability implements LoggerAwareInterface
             // For every SCORE_CHARS_IN_PARAGRAPH (default:100) characters in this paragraph, add another point. Up to 3 points.
             $contentScore += min(floor(mb_strlen($innerText) / self::SCORE_CHARS_IN_PARAGRAPH), 3);
             // For every SCORE_WORDS_IN_PARAGRAPH (default:20) words in this paragraph, add another point. Up to 3 points.
-            //$contentScore += min(floor($this->getWordCount($innerText) / self::SCORE_WORDS_IN_PARAGRAPH), 3);
+            // $contentScore += min(floor($this->getWordCount($innerText) / self::SCORE_WORDS_IN_PARAGRAPH), 3);
 
             foreach ($ancestors as $level => $ancestor) {
                 if (!$ancestor->nodeName || !$ancestor->parentNode) {
@@ -1079,11 +1079,6 @@ class Readability implements LoggerAwareInterface
                 }
             }
 
-            $candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
-
-            for ($c = $candidates->length - 1; $c >= 0; --$c) {
-                $node = $candidates->item($c);
-            }
             unset($candidates);
         }
 
@@ -1121,9 +1116,11 @@ class Readability implements LoggerAwareInterface
             }
         }
 
-        $topCandidates = array_filter($topCandidates, function ($v, $idx) {
-            return 0 === $idx || null !== $v;
-        }, \ARRAY_FILTER_USE_BOTH);
+        $topCandidates = array_filter(
+            $topCandidates,
+            fn ($v, $idx) => 0 === $idx || null !== $v,
+            \ARRAY_FILTER_USE_BOTH
+        );
         $topCandidate = $topCandidates[0];
 
         /*
@@ -1211,7 +1208,7 @@ class Readability implements LoggerAwareInterface
         if (0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'tr')) {
             $up = $topCandidate;
 
-            if ($up->parentNode instanceof DOMElement) {
+            if ($up->parentNode instanceof \DOMElement) {
                 $up = $up->parentNode;
 
                 if (0 === strcasecmp($up->tagName, 'table')) {
@@ -1232,11 +1229,6 @@ class Readability implements LoggerAwareInterface
         $parentOfTopCandidate = $topCandidate->parentNode;
         $siblingNodes = $parentOfTopCandidate->childNodes;
 
-        if (0 === $siblingNodes->length) {
-            $siblingNodes = new \stdClass();
-            $siblingNodes->length = 0;
-        }
-
         for ($s = 0, $sl = $siblingNodes->length; $s < $sl; ++$s) {
             $siblingNode = $siblingNodes->item($s);
             $siblingNodeName = $siblingNode->nodeName;
@@ -1292,8 +1284,8 @@ class Readability implements LoggerAwareInterface
 
                 // To ensure a node does not interfere with readability styles, remove its classnames & ids.
                 // Now done via RegExp post_filter.
-                //$nodeToAppend->removeAttribute('class');
-                //$nodeToAppend->removeAttribute('id');
+                // $nodeToAppend->removeAttribute('class');
+                // $nodeToAppend->removeAttribute('id');
                 // Append sibling and subtract from our list as appending removes a node.
                 $articleContent->appendChild($nodeToAppend);
             }
@@ -1340,7 +1332,7 @@ class Readability implements LoggerAwareInterface
      * Get an element weight by attribute.
      * Uses regular expressions to tell if this element looks good or bad.
      */
-    protected function weightAttribute(DOMElement $element, string $attribute): int
+    protected function weightAttribute(\DOMElement $element, string $attribute): int
     {
         if (!$element->hasAttribute($attribute)) {
             return 0;
@@ -1391,7 +1383,10 @@ class Readability implements LoggerAwareInterface
         $this->logger->debug('Parsing URL: ' . $this->url);
 
         if ($this->url) {
-            $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', (string) parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
         }
 
         mb_internal_encoding('UTF-8');
@@ -1427,7 +1422,7 @@ class Readability implements LoggerAwareInterface
             unset($tidy);
         }
 
-        $this->html = mb_convert_encoding((string) $this->html, 'HTML-ENTITIES', 'UTF-8');
+        $this->html = self::entitizeNonAscii((string) $this->html);
 
         if ('html5lib' === $this->parser || 'html5' === $this->parser) {
             $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1443,14 +1438,14 @@ class Readability implements LoggerAwareInterface
             libxml_use_internal_errors(false);
         }
 
-        $this->dom->registerNodeClass(DOMElement::class, \Readability\JSLikeHTMLElement::class);
+        $this->dom->registerNodeClass(\DOMElement::class, JSLikeHTMLElement::class);
     }
 
-    private function getAncestors(DOMElement $node, int $maxDepth = 0): array
+    private function getAncestors(\DOMElement $node, int $maxDepth = 0): array
     {
         $ancestors = [];
         $i = 0;
-        while ($node->parentNode instanceof DOMElement) {
+        while ($node->parentNode instanceof \DOMElement) {
             $ancestors[] = $node->parentNode;
             if (++$i === $maxDepth) {
                 break;
@@ -1465,21 +1460,38 @@ class Readability implements LoggerAwareInterface
     {
         return \XML_TEXT_NODE === $node->nodeType
             || \in_array(strtoupper($node->nodeName), $this->phrasingElements, true)
-            || (\in_array(strtoupper($node->nodeName), ['A', 'DEL', 'INS'], true) && !\in_array(false, array_map(function ($c) {
-                return $this->isPhrasingContent($c);
-            }, iterator_to_array($node->childNodes)), true));
+            || (
+                \in_array(strtoupper($node->nodeName), ['A', 'DEL', 'INS'], true)
+                && !\in_array(
+                    false,
+                    array_map(
+                        fn ($c) => $this->isPhrasingContent($c),
+                        iterator_to_array($node->childNodes)
+                    ),
+                    true
+                )
+            );
     }
 
-    private function hasSingleTagInsideElement(DOMElement $node, string $tag): bool
+    /**
+     * Checks if `$node` has only whitespace and a single element with `$tag` for the tag name.
+     * Returns false if `$node` contains non-empty text nodes
+     * or if it contains no element with given tag or more than 1 element.
+     */
+    private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
     {
-        if (1 !== $node->childNodes->length || $node->childNodes->item(0)->nodeName !== $tag) {
+        $childNodes = iterator_to_array($node->childNodes);
+        $children = array_filter($childNodes, fn ($childNode) => $childNode instanceof \DOMElement);
+
+        // There should be exactly 1 element child with given tag
+        if (1 !== \count($children) || $children[0]->nodeName !== $tag) {
             return false;
         }
 
-        $a = array_filter(iterator_to_array($node->childNodes), function ($childNode) {
-            return $childNode instanceof \DOMText &&
-                preg_match($this->regexps['hasContent'], $this->getInnerText($childNode));
-        });
+        $a = array_filter(
+            $childNodes,
+            fn ($childNode) => $childNode instanceof \DOMText && preg_match($this->regexps['hasContent'], $this->getInnerText($childNode))
+        );
 
         return 0 === \count($a);
     }
@@ -1490,11 +1502,29 @@ class Readability implements LoggerAwareInterface
      * Tidy must be configured to not clean the input for this function to
      * work as expected, see $this->tidy_config['clean']
      */
-    private function isNodeVisible(DOMElement $node): bool
+    private function isNodeVisible(\DOMElement $node): bool
     {
-        return !($node->hasAttribute('style')
-                    && preg_match($this->regexps['isNotVisible'], $node->getAttribute('style'))
-                )
-                && !$node->hasAttribute('hidden');
+        return !(
+            $node->hasAttribute('style')
+            && preg_match($this->regexps['isNotVisible'], $node->getAttribute('style'))
+        )
+        && !$node->hasAttribute('hidden');
+    }
+
+    /**
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
+     *
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     *
+     * @param string $html UTF-8 encoded document
+     */
+    private static function entitizeNonAscii(string $html): string
+    {
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];
+
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
     }
 }

tests/ReadabilityTest.php

@@ -24,7 +24,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructHtml5Parser(): void
@@ -33,7 +33,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
     }
 
@@ -46,7 +46,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertTrue($readability->tidied);
     }
@@ -60,7 +60,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertSame('', $readability->original_html);
         $this->assertFalse($readability->tidied);
 
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructSimpleWithoutTidy(): void
@@ -69,7 +69,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertFalse($readability->tidied);
     }
@@ -115,7 +115,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testInitDiv(): void
     {
         $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -129,7 +128,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithFootnotes(): void
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -146,7 +144,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testStandardClean(): void
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -163,7 +160,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithIframe(): void
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -178,7 +174,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithArticle(): void
     {
         $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -193,7 +188,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithAside(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -208,7 +202,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClasses(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -223,7 +216,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClassesWithoutLightClean(): void
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -239,7 +231,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithTd(): void
     {
         $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -252,7 +243,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithSameClasses(): void
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -266,7 +256,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithScript(): void
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -280,7 +269,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitle(): void
     {
         $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -294,7 +282,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDash(): void
     {
         $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -308,7 +295,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDoubleDot(): void
     {
         $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -322,7 +308,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleTooShortUseH1(): void
     {
         $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -335,48 +320,55 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
 
     public function testAutoClosingIframeNotThrowingException(): void
     {
-        error_reporting(\E_ALL | \E_STRICT);
-        ini_set('display_errors', '1');
+        $oldErrorReporting = error_reporting(\E_ALL);
+        $oldDisplayErrors = ini_set('display_errors', '1');
         // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline, array $errcontext) {
+        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
             throw new \Exception($errstr, $errno);
-        }, \E_ALL | \E_STRICT);
+        });
 
-        $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-            <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
+        try {
+            $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+                <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
 
-            <head profile="http://gmpg.org/xfn/11">
-            <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+                <head profile="http://gmpg.org/xfn/11">
+                <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+                <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
-            </head>
-            <body class="single single-post postid-22030 single-format-standard">
-              <div id="wrapper">
-                <div id="content">
-                  <div class="post-22030 post type-post status-publish format-standard has-post-thumbnail hentry category-video category-reviews tag-193" id="post-22030">
-                    <h1>3D Touch &#8212; будущее мобильных игр</h1>
-                    <div class="postdate">Автор:  <strong>Сергей Пак</strong> | Просмотров: 1363 | Опубликовано:  14 сентября 2015 </div>
-                    <div class="entry">
-                      <p>Компания Apple представила новую технологию 3D Touch, которая является прямым потомком более ранней версии Force Touch &#8212; последняя, напомним, используется сейчас в трекпадах Macbook Pro и Macbook 2015. Теперь управлять устройством стало в разы проще, и Force Touch открывает перед пользователями новые возможности, но при этом 3D Touch &#8212; это про другое. Дело в том, что теперь и на мобильных устройствах интерфейс будет постепенно меняться, кардинальные перемены ждут мобильный гейминг, потому что здесь разработчики действительно могут разгуляться.<span id="more-22030"></span></p>
-                      <p><iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"></iframe></p>
-                      <p>Итак, просто представьте себе, что iPhone 6S &#8212; это, по большому счету, отличная игровая приставка, которую вы носите с собой, а еще она может выдавать невероятной красоты картинку. Но проблема заключается, пожалуй, в том, что управлять персонажем в играх довольно трудно &#8212; он неповоротлив, обладает заторможенной реакцией, а игровой клиент зачастую требует перегруза интерфейса для того, чтобы обеспечить максимально большое количество возможностей. Благодаря трехуровневому нажатию можно избавиться от лишних кнопок и обеспечить более качественный обзор местности, и при этом пользователь будет закрывать пальцами минимальное пространство.</p>
+                </head>
+                <body class="single single-post postid-22030 single-format-standard">
+                  <div id="wrapper">
+                    <div id="content">
+                      <div class="post-22030 post type-post status-publish format-standard has-post-thumbnail hentry category-video category-reviews tag-193" id="post-22030">
+                        <h1>3D Touch &#8212; будущее мобильных игр</h1>
+                        <div class="postdate">Автор:  <strong>Сергей Пак</strong> | Просмотров: 1363 | Опубликовано:  14 сентября 2015 </div>
+                        <div class="entry">
+                          <p>Компания Apple представила новую технологию 3D Touch, которая является прямым потомком более ранней версии Force Touch &#8212; последняя, напомним, используется сейчас в трекпадах Macbook Pro и Macbook 2015. Теперь управлять устройством стало в разы проще, и Force Touch открывает перед пользователями новые возможности, но при этом 3D Touch &#8212; это про другое. Дело в том, что теперь и на мобильных устройствах интерфейс будет постепенно меняться, кардинальные перемены ждут мобильный гейминг, потому что здесь разработчики действительно могут разгуляться.<span id="more-22030"></span></p>
+                          <p><iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"></iframe></p>
+                          <p>Итак, просто представьте себе, что iPhone 6S &#8212; это, по большому счету, отличная игровая приставка, которую вы носите с собой, а еще она может выдавать невероятной красоты картинку. Но проблема заключается, пожалуй, в том, что управлять персонажем в играх довольно трудно &#8212; он неповоротлив, обладает заторможенной реакцией, а игровой клиент зачастую требует перегруза интерфейса для того, чтобы обеспечить максимально большое количество возможностей. Благодаря трехуровневому нажатию можно избавиться от лишних кнопок и обеспечить более качественный обзор местности, и при этом пользователь будет закрывать пальцами минимальное пространство.</p>
+                        </div>
+                      </div>
                     </div>
                   </div>
-                </div>
-              </div>
-            </body>
-            </html>';
+                </body>
+                </html>';
 
-        $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-        $readability->debug = true;
+            $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
 
-        $res = $readability->init();
+            $res = $readability->init();
 
-        $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
-        $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
-        $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
+            $this->assertTrue($res);
+            $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+            $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+            $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
+            $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
+        } finally {
+            restore_error_handler();
+            if (false !== $oldDisplayErrors) {
+                ini_set('display_errors', $oldDisplayErrors);
+            }
+            error_reporting($oldErrorReporting);
+        }
     }
 
     /**
@@ -429,7 +421,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             </html>';
 
         $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;
 
         $res = $readability->init();
 
@@ -466,7 +457,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/childNodeGoesNull.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -479,7 +469,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/keepFootnotes.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -493,7 +482,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = (string) file_get_contents('tests/fixtures/wipedBody.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -532,7 +520,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testVisibleNode(string $content, bool $shouldBeVisible): void
     {
         $readability = $this->getReadability($content, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         if ($shouldBeVisible) {
@@ -542,7 +529,65 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         }
     }
 
-    private function getReadability(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle(): void
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang(): array
+    {
+        return [
+            'meta' => [
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'head' => [
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
+                false,
+            ],
+            'fragment' => [
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
+                '',
+                // tidy would add <html>.
+                false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHtmlLang
+     */
+    public function testHtmlLang(string $html, string $lang, bool $useTidy = true): void
+    {
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
+    }
+
+    private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
     {
         $readability = new Readability($html, $url, $parser, $useTidy);