Merge pull request #110 from j0k3r/fix/psr-monolog

[1.x] Allow new psr/log & monolog

.gitattributes

@@ -6,6 +6,5 @@
 /.php_cs export-ignore
 /phpunit.xml.dist export-ignore
 /phpstan.neon export-ignore
-/rector.php export-ignore
 /.github export-ignore
 /tests export-ignore

.github/workflows/coding-standards.yml

@@ -4,23 +4,28 @@ on:
   pull_request:
     branches:
       - master
+      - 1.x
   push:
     branches:
       - master
+      - 1.x
+
+env:
+  SYMFONY_PHPUNIT_VERSION: 7.5
 
 jobs:
   coding-standards:
     name: "CS Fixer & PHPStan"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
         php:
-          - "7.2"
+          - "7.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
 
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
@@ -32,8 +37,11 @@ jobs:
         env:
           COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: "Add PHPStan"
+        run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"
+
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 

.github/workflows/continuous-integration.yml

@@ -4,9 +4,11 @@ on:
   pull_request:
     branches:
       - "master"
+      - "1.x"
   push:
     branches:
       - "master"
+      - "1.x"
 
 env:
   fail-fast: true
@@ -14,7 +16,7 @@ env:
 jobs:
   phpunit:
     name: "PHPUnit (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
@@ -26,10 +28,11 @@ jobs:
           - "8.1"
           - "8.2"
           - "8.3"
+          - "8.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -48,7 +51,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -60,7 +63,7 @@ jobs:
 
   phpunit-coverage:
     name: "PHPUnit coverage (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
@@ -69,7 +72,7 @@ jobs:
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -88,7 +91,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -111,16 +114,16 @@ jobs:
 
   phpunit-lowest:
     name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
         php:
-          - "7.2"
+          - "7.4"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
 
@@ -139,7 +142,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v2"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
           dependency-versions: "lowest"

.gitignore

@@ -2,5 +2,5 @@ vendor/
 coverage/
 composer.lock
 .php_cs.cache
-.php-cs-fixer.cache
 .phpunit.result.cache
+phpstan.neon

.php-cs-fixer.php

@@ -26,6 +26,10 @@ return (new PhpCsFixer\Config())
         'strict_comparison' => true,
         'strict_param' => true,
         'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
+        // Pulled in by @Symfony with `const` but const visibility requires PHP ≥ 7.1
+        'visibility_required' => ['elements' => ['method', 'property']],
     ])
     ->setFinder($finder)
 ;

.scrutinizer.yml

@@ -0,0 +1,12 @@
+tools:
+    external_code_coverage: false
+
+build:
+    nodes:
+        analysis:
+            tests:
+                override:
+                    - php-scrutinizer-run
+            environment:
+                php:
+                    version: 7.2

README.md

@@ -5,7 +5,7 @@
 [![Total Downloads](https://poser.pugx.org/j0k3r/php-readability/downloads)](https://packagist.org/packages/j0k3r/php-readability)
 [![License](https://poser.pugx.org/j0k3r/php-readability/license)](https://packagist.org/packages/j0k3r/php-readability)
 
-This is an extract of the Readability class from this [full-text-rss](https://github.com/Dither/full-text-rss) fork. It can be defined as a better version of the original [php-readability](https://bitbucket.org/fivefilters/php-readability).
+This is an extract of the Readability class from this [full-text-rss](https://github.com/Dither/full-text-rss) fork. It can be defined as a better version of the original [php-readability](https://bitbucket.org/fivefilters/php-readability/overview).
 
 ## Differences
 

composer.json

@@ -24,18 +24,15 @@
         "role": "Developer (original JS version)"
     }],
     "require": {
-        "php": ">=7.2.0",
+        "php": ">=7.2",
         "ext-mbstring": "*",
         "psr/log": "^1.0.1 || ^2.0 || ^3.0",
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
-        "friendsofphp/php-cs-fixer": "^3.0",
+        "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24|^2.1",
+        "monolog/monolog": "^1.24|^2.1|^3.0",
-        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0",
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0"
-        "phpstan/phpstan": "^1.3",
-        "phpstan/phpstan-phpunit": "^1.0",
-        "rector/rector": "^0.15.0"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -49,7 +46,6 @@
     "scripts": {
         "fix": "php-cs-fixer fix --verbose --diff",
         "phpstan": "phpstan analyze --memory-limit 512M",
-        "rector": "rector process",
         "test": "simple-phpunit -v"
     }
 }

phpstan.dist.neon

@@ -8,8 +8,6 @@ parameters:
     bootstrapFiles:
         - vendor/bin/.phpunit/phpunit/vendor/autoload.php
 
-    checkMissingIterableValueType: false
-
 includes:
     - vendor/phpstan/phpstan-phpunit/extension.neon
     - vendor/phpstan/phpstan-phpunit/rules.neon

phpunit.xml.dist

@@ -11,7 +11,7 @@
     >
 
     <testsuites>
-        <testsuite name="Readability">
+        <testsuite name="Readability Test Suite">
             <directory>./tests/</directory>
         </testsuite>
     </testsuites>

rector.php

@@ -1,31 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-use Rector\Config\RectorConfig;
-use Rector\Core\ValueObject\PhpVersion;
-use Rector\Set\ValueObject\LevelSetList;
-
-return static function (RectorConfig $rectorConfig): void {
-    // paths to refactor; solid alternative to CLI arguments
-    $rectorConfig->paths([
-        __DIR__ . '/src',
-        __DIR__ . '/tests',
-    ]);
-
-    // Path to phpstan with extensions, that PHPSTan in Rector uses to determine types
-    $rectorConfig->phpstanConfig(__DIR__ . '/phpstan.neon');
-
-    $rectorConfig->bootstrapFiles([
-        __DIR__ . '/vendor/bin/.phpunit/phpunit/vendor/autoload.php',
-        __DIR__ . '/vendor/autoload.php',
-    ]);
-
-    // Define what rule sets will be applied
-    $rectorConfig->sets([
-        LevelSetList::UP_TO_PHP_72,
-    ]);
-
-    // is your PHP version different from the one your refactor to?
-    $rectorConfig->phpVersion(PhpVersion::PHP_72);
-};

src/Readability.php

@@ -7,25 +7,68 @@ use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 
+/**
+ * Arc90's Readability ported to PHP for FiveFilters.org
+ * Based on readability.js version 1.7.1 (without multi-page support)
+ * ------------------------------------------------------
+ * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
+ * Arc90's project URL: http://lab.arc90.com/experiments/readability/
+ * JS Source: http://code.google.com/p/arc90labs-readability
+ * Ported by: Keyvan Minoukadeh, http://www.keyvan.net
+ * Modded by: Dither, https://dithersky.wordpress.com
+ * More information: http://fivefilters.org/content-only/
+ * License: Apache License, Version 2.0
+ * Requires: PHP version 5.2.0+
+ * Date: 2013-08-02.
+ *
+ * Differences between the PHP port and the original
+ * ------------------------------------------------------
+ * Arc90's Readability is designed to run in the browser. It works on the DOM
+ * tree (the parsed HTML) after the page's CSS styles have been applied and
+ * Javascript code executed. This PHP port does not run inside a browser.
+ * We use PHP's ability to parse HTML to build our DOM tree, but we cannot
+ * rely on CSS or Javascript support. As such, the results will not always
+ * match Arc90's Readability. (For example, if a web page contains CSS style
+ * rules or Javascript code which hide certain HTML elements from display,
+ * Arc90's Readability will dismiss those from consideration but our PHP port,
+ * unable to understand CSS or Javascript, will not know any better.)
+ *
+ * Another significant difference is that the aim of Arc90's Readability is
+ * to re-present the main content block of a given web page so users can
+ * read it more easily in their browsers. Correct identification, clean up,
+ * and separation of the content block is only a part of this process.
+ * This PHP port is only concerned with this part, it does not include code
+ * that relates to presentation in the browser - Arc90 already do
+ * that extremely well, and for PDF output there's FiveFilters.org's
+ * PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
+ *
+ * Finally, this class contains methods that might be useful for developers
+ * working on HTML document fragments. So without deviating too much from
+ * the original code (which I don't want to do because it makes debugging
+ * and updating more difficult), I've tried to make it a little more
+ * developer friendly. You should be able to use the methods here on
+ * existing DOMElement objects without passing an entire HTML document to
+ * be parsed.
+ */
 class Readability implements LoggerAwareInterface
 {
     // flags
-    public const FLAG_STRIP_UNLIKELYS = 1;
+    const FLAG_STRIP_UNLIKELYS = 1;
-    public const FLAG_WEIGHT_ATTRIBUTES = 2;
+    const FLAG_WEIGHT_ATTRIBUTES = 2;
-    public const FLAG_CLEAN_CONDITIONALLY = 4;
+    const FLAG_CLEAN_CONDITIONALLY = 4;
-    public const FLAG_DISABLE_PREFILTER = 8;
+    const FLAG_DISABLE_PREFILTER = 8;
-    public const FLAG_DISABLE_POSTFILTER = 16;
+    const FLAG_DISABLE_POSTFILTER = 16;
     // constants
-    public const SCORE_CHARS_IN_PARAGRAPH = 100;
+    const SCORE_CHARS_IN_PARAGRAPH = 100;
-    public const SCORE_WORDS_IN_PARAGRAPH = 20;
+    const SCORE_WORDS_IN_PARAGRAPH = 20;
-    public const GRANDPARENT_SCORE_DIVISOR = 2;
+    const GRANDPARENT_SCORE_DIVISOR = 2;
-    public const MIN_PARAGRAPH_LENGTH = 20;
+    const MIN_PARAGRAPH_LENGTH = 20;
-    public const MIN_COMMAS_IN_PARAGRAPH = 6;
+    const MIN_COMMAS_IN_PARAGRAPH = 6;
-    public const MIN_ARTICLE_LENGTH = 200;
+    const MIN_ARTICLE_LENGTH = 200;
-    public const MIN_NODE_LENGTH = 80;
+    const MIN_NODE_LENGTH = 80;
-    public const MAX_LINK_DENSITY = 0.25;
+    const MAX_LINK_DENSITY = 0.25;
     public $convertLinksToFootnotes = false;
-    public $revertForcedParagraphElements = false;
+    public $revertForcedParagraphElements = true;
     public $articleTitle;
     public $articleContent;
     public $original_html;
@@ -46,27 +89,14 @@ class Readability implements LoggerAwareInterface
      * Defined up here so we don't instantiate them repeatedly in loops.
      */
     public $regexps = [
-        'unlikelyCandidates' => '/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i',
+        'unlikelyCandidates' => '/display\s*:\s*none|ignore|\binfos?\b|annoy|clock|date|time|author|intro|hidd?e|about|archive|\bprint|bookmark|tags|tag-list|share|search|social|robot|published|combx|comment|mast(?:head)|subscri|community|category|disqus|extra|head|head(?:er|note)|floor|foot(?:er|note)|menu|tool\b|function|nav|remark|rss|shoutbox|widget|meta|banner|sponsor|adsense|inner-?ad|ad-|sponsor|\badv\b|\bads\b|agr?egate?|pager|sidebar|popup|tweet|twitter/i',
-        'okMaybeItsACandidate' => '/article\b|contain|\bcontent|column|general|detail|shadow|lightbox|blog|body|entry|main|page|footnote|element/i',
+        'okMaybeItsACandidate' => '/article\b|contain|\bcontent|column|general|detail|shadow|lightbox|blog|body|entry|main|page|footnote/i',
         'positive' => '/read|full|article|body|\bcontent|contain|entry|main|markdown|media|page|attach|pagination|post|text|blog|story/i',
         'negative' => '/bottom|stat|info|discuss|e[\-]?mail|comment|reply|log.{2}(n|ed)|sign|single|combx|com-|contact|_nav|link|media|promo|\bad-|related|scroll|shoutbox|sidebar|sponsor|shopping|teaser|recommend/i',
         'divToPElements' => '/<(?:blockquote|header|section|code|div|article|footer|aside|img|p|pre|dl|ol|ul)/mi',
         'killBreaks' => '/(<br\s*\/?>([ \r\n\s]|&nbsp;?)*)+/',
         'media' => '!//(?:[^\.\?/]+\.)?(?:youtu(?:be)?|giphy|soundcloud|dailymotion|vimeo|pornhub|xvideos|twitvid|rutube|openload\.co|viddler)\.(?:com|be|org|net)/!i',
         'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i',
-        'hasContent' => '/\S$/',
-        'isNotVisible' => '/display\s*:\s*none/',
-    ];
-    public $defaultTagsToScore = ['section', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'td', 'pre'];
-    // The commented out elements qualify as phrasing content but tend to be
-    // removed by readability when put into paragraphs, so we ignore them here.
-    public $phrasingElements = [
-        // "CANVAS", "IFRAME", "SVG", "VIDEO",
-        'ABBR', 'AUDIO', 'B', 'BDO', 'BR', 'BUTTON', 'CITE', 'CODE', 'DATA',
-        'DATALIST', 'DFN', 'EM', 'EMBED', 'I', 'IMG', 'INPUT', 'KBD', 'LABEL',
-        'MARK', 'MATH', 'METER', 'NOSCRIPT', 'OBJECT', 'OUTPUT', 'PROGRESS', 'Q',
-        'RUBY', 'SAMP', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN', 'STRONG', 'SUB',
-        'SUP', 'TEXTAREA', 'TIME', 'VAR', 'WBR',
     ];
     public $tidy_config = [
         'tidy-mark' => false,
@@ -75,7 +105,7 @@ class Readability implements LoggerAwareInterface
         'numeric-entities' => false,
         // 'preserve-entities' => true,
         'break-before-br' => false,
-        'clean' => false,
+        'clean' => true,
         'output-xhtml' => true,
         'logical-emphasis' => true,
         'show-body-only' => false,
@@ -140,16 +170,17 @@ class Readability implements LoggerAwareInterface
      * @param string $html     UTF-8 encoded string
      * @param string $url      URL associated with HTML (for footnotes)
      * @param string $parser   Which parser to use for turning raw HTML into a DOMDocument
-     * @param bool   $useTidy Use tidy
+     * @param bool   $use_tidy Use tidy
      */
-    public function __construct(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true)
+    public function __construct($html, $url = null, $parser = 'libxml', $use_tidy = true)
     {
         $this->url = $url;
         $this->html = $html;
         $this->parser = $parser;
-        $this->useTidy = $useTidy && \function_exists('tidy_parse_string');
+        $this->useTidy = $use_tidy && \function_exists('tidy_parse_string');
 
         $this->logger = new NullLogger();
+        $this->loadHtml();
     }
 
     public function setLogger(LoggerInterface $logger): void
@@ -183,7 +214,7 @@ class Readability implements LoggerAwareInterface
      * @param string $filter   RegExp for replace
      * @param string $replacer Replacer
      */
-    public function addPreFilter(string $filter, string $replacer = ''): void
+    public function addPreFilter($filter, $replacer = '')
     {
         $this->pre_filters[$filter] = $replacer;
     }
@@ -194,7 +225,7 @@ class Readability implements LoggerAwareInterface
      * @param string $filter   RegExp for replace
      * @param string $replacer Replacer
      */
-    public function addPostFilter(string $filter, string $replacer = ''): void
+    public function addPostFilter($filter, $replacer = '')
     {
         $this->post_filters[$filter] = $replacer;
     }
@@ -211,10 +242,8 @@ class Readability implements LoggerAwareInterface
      *
      * @return bool true if we found content, false otherwise
      */
-    public function init(): bool
+    public function init()
     {
-        $this->loadHtml();
-
         if (!isset($this->dom->documentElement)) {
             return false;
         }
@@ -279,7 +308,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Run any post-process modifications to article content as necessary.
      */
-    public function postProcessContent(\DOMElement $articleContent): void
+    public function postProcessContent(\DOMElement $articleContent)
     {
         if ($this->convertLinksToFootnotes && !preg_match('/\bwiki/', $this->url)) {
             $this->addFootnotes($articleContent);
@@ -291,7 +320,7 @@ class Readability implements LoggerAwareInterface
      *
      * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
      */
-    public function addFootnotes(\DOMElement $articleContent): void
+    public function addFootnotes(\DOMElement $articleContent)
     {
         $footnotesWrapper = $this->dom->createElement('footer');
         $footnotesWrapper->setAttribute('class', 'readability-footnotes');
@@ -353,7 +382,7 @@ class Readability implements LoggerAwareInterface
      * Prepare the article node for display. Clean out any inline styles,
      * iframes, forms, strip extraneous <p> tags, etc.
      */
-    public function prepArticle(\DOMNode $articleContent): void
+    public function prepArticle(\DOMNode $articleContent)
     {
         if (!$articleContent instanceof \DOMElement) {
             return;
@@ -458,8 +487,10 @@ class Readability implements LoggerAwareInterface
      * @param \DOMElement $e
      * @param bool        $normalizeSpaces (default: true)
      * @param bool        $flattenLines    (default: false)
+     *
+     * @return string
      */
-    public function getInnerText($e, bool $normalizeSpaces = true, bool $flattenLines = false): string
+    public function getInnerText($e, $normalizeSpaces = true, $flattenLines = false)
     {
         if (null === $e || !isset($e->textContent) || '' === $e->textContent) {
             return '';
@@ -468,11 +499,9 @@ class Readability implements LoggerAwareInterface
         $textContent = trim($e->textContent);
 
         if ($flattenLines) {
-            return (string) mb_ereg_replace('(?:[\r\n](?:\s|&nbsp;)*)+', '', $textContent);
+            $textContent = mb_ereg_replace('(?:[\r\n](?:\s|&nbsp;)*)+', '', $textContent);
-        }
+        } elseif ($normalizeSpaces) {
-
+            $textContent = mb_ereg_replace('\s\s+', ' ', $textContent);
-        if ($normalizeSpaces) {
-            return (string) mb_ereg_replace('\s\s+', ' ', $textContent);
         }
 
         return $textContent;
@@ -480,31 +509,43 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Remove the style attribute on every $e and under.
+     *
+     * @param \DOMElement $e
      */
-    public function cleanStyles(\DOMElement $e): void
+    public function cleanStyles($e)
     {
-        if (\is_object($e)) {
+        if (!\is_object($e)) {
+            return;
+        }
+
         $elems = $e->getElementsByTagName('*');
 
         foreach ($elems as $elem) {
             $elem->removeAttribute('style');
         }
     }
-    }
 
     /**
      * Get comma number for a given text.
+     *
+     * @param string $text
+     *
+     * @return int
      */
-    public function getCommaCount(string $text): int
+    public function getCommaCount($text)
     {
-        return \count(explode(',', $text));
+        return substr_count($text, ',');
     }
 
     /**
      * Get words number for a given text if words separated by a space.
      * Input string should be normalized.
+     *
+     * @param string $text
+     *
+     * @return int
      */
-    public function getWordCount(string $text): int
+    public function getWordCount($text)
     {
         return substr_count($text, ' ');
     }
@@ -513,8 +554,12 @@ class Readability implements LoggerAwareInterface
      * Get the density of links as a percentage of the content
      * This is the amount of text that is inside a link divided by the total text in the node.
      * Can exclude external references to differentiate between simple text and menus/infoblocks.
+     *
+     * @param bool $excludeExternal
+     *
+     * @return int
      */
-    public function getLinkDensity(\DOMElement $e, bool $excludeExternal = false): float
+    public function getLinkDensity(\DOMElement $e, $excludeExternal = false)
     {
         $links = $e->getElementsByTagName('a');
         $textLength = mb_strlen($this->getInnerText($e, true, true));
@@ -536,8 +581,10 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Get an element relative weight.
+     *
+     * @return int
      */
-    public function getWeight(\DOMElement $e): int
+    public function getWeight(\DOMElement $e)
     {
         if (!$this->flagIsActive(self::FLAG_WEIGHT_ATTRIBUTES)) {
             return 0;
@@ -555,7 +602,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Remove extraneous break tags from a node.
      */
-    public function killBreaks(\DOMElement $node): void
+    public function killBreaks(\DOMElement $node)
     {
         $html = $node->getInnerHTML();
         $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
@@ -567,8 +614,10 @@ class Readability implements LoggerAwareInterface
      * (Unless it's a youtube/vimeo video. People love movies.).
      *
      * Updated 2012-09-18 to preserve youtube/vimeo iframes
+     *
+     * @param string $tag
      */
-    public function clean(\DOMElement $e, string $tag): void
+    public function clean(\DOMElement $e, $tag)
     {
         $targetList = $e->getElementsByTagName($tag);
         $isEmbed = ('audio' === $tag || 'video' === $tag || 'iframe' === $tag || 'object' === $tag || 'embed' === $tag);
@@ -599,8 +648,10 @@ class Readability implements LoggerAwareInterface
      * Clean an element of all tags of type "tag" if they look fishy.
      * "Fishy" is an algorithm based on content length, classnames,
      * link density, number of images & embeds, etc.
+     *
+     * @param string $tag
      */
-    public function cleanConditionally(\DOMElement $e, string $tag): void
+    public function cleanConditionally(\DOMElement $e, $tag)
     {
         if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
             return;
@@ -621,9 +672,6 @@ class Readability implements LoggerAwareInterface
             $contentScore = ($node->hasAttribute('readability')) ? (int) $node->getAttribute('readability') : 0;
             $this->logger->debug('Start conditional cleaning of ' . $node->getNodePath() . ' (class=' . $node->getAttribute('class') . '; id=' . $node->getAttribute('id') . ')' . (($node->hasAttribute('readability')) ? (' with score ' . $node->getAttribute('readability')) : ''));
 
-            // XXX Incomplete implementation
-            $isList = \in_array($node->tagName, ['ul', 'ol'], true);
-
             if ($weight + $contentScore < 0) {
                 $this->logger->debug('Removing...');
                 $node->parentNode->removeChild($node);
@@ -658,16 +706,16 @@ class Readability implements LoggerAwareInterface
                 $toRemove = false;
 
                 if ($this->lightClean) {
-                    if (!$isList && $li > $p) {
+                    if ($li > $p && 'ul' !== $tag && 'ol' !== $tag) {
                         $this->logger->debug(' too many <li> elements, and parent is not <ul> or <ol>');
                         $toRemove = true;
                     } elseif ($input > floor($p / 3)) {
                         $this->logger->debug(' too many <input> elements');
                         $toRemove = true;
-                    } elseif (!$isList && $contentLength < 6 && (0 === $embedCount && (0 === $img || $img > 2))) {
+                    } elseif ($contentLength < 6 && (0 === $embedCount && (0 === $img || $img > 2))) {
                         $this->logger->debug(' content length less than 6 chars, 0 embeds and either 0 images or more than 2 images');
                         $toRemove = true;
-                    } elseif (!$isList && $weight < 25 && $linkDensity > 0.25) {
+                    } elseif ($weight < 25 && $linkDensity > 0.25) {
                         $this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.25');
                         $toRemove = true;
                     } elseif ($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
@@ -681,16 +729,16 @@ class Readability implements LoggerAwareInterface
                     if ($img > $p) {
                         $this->logger->debug(' more image elements than paragraph elements');
                         $toRemove = true;
-                    } elseif (!$isList && $li > $p) {
+                    } elseif ($li > $p && 'ul' !== $tag && 'ol' !== $tag) {
                         $this->logger->debug('  too many <li> elements, and parent is not <ul> or <ol>');
                         $toRemove = true;
                     } elseif ($input > floor($p / 3)) {
                         $this->logger->debug('  too many <input> elements');
                         $toRemove = true;
-                    } elseif (!$isList && $contentLength < 10 && (0 === $img || $img > 2)) {
+                    } elseif ($contentLength < 10 && (0 === $img || $img > 2)) {
                         $this->logger->debug('  content length less than 10 chars and 0 images, or more than 2 images');
                         $toRemove = true;
-                    } elseif (!$isList && $weight < 25 && $linkDensity > 0.2) {
+                    } elseif ($weight < 25 && $linkDensity > 0.2) {
                         $this->logger->debug('  weight is ' . $weight . ' lower than 0 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.2');
                         $toRemove = true;
                     } elseif ($weight >= 25 && $linkDensity > 0.5) {
@@ -713,7 +761,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Clean out spurious headers from an Element. Checks things like classnames and link density.
      */
-    public function cleanHeaders(\DOMElement $e): void
+    public function cleanHeaders(\DOMElement $e)
     {
         for ($headerIndex = 1; $headerIndex < 3; ++$headerIndex) {
             $headers = $e->getElementsByTagName('h' . $headerIndex);
@@ -728,26 +776,57 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Check if the given flag is active.
+     *
+     * @param int $flag
+     *
+     * @return bool
      */
-    public function flagIsActive(int $flag): bool
+    public function flagIsActive($flag)
     {
         return ($this->flags & $flag) > 0;
     }
 
     /**
      * Add a flag.
+     *
+     * @param int $flag
      */
-    public function addFlag(int $flag): void
+    public function addFlag($flag)
     {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
     }
 
     /**
      * Remove a flag.
+     *
+     * @param int $flag
      */
-    public function removeFlag(int $flag): void
+    public function removeFlag($flag)
+    {
+        $this->flags &= ~$flag;
+    }
+
+    /**
+     * Debug.
+     *
+     * @deprecated use $this->logger->debug() instead
+     *
+     * @codeCoverageIgnore
+     */
+    protected function dbg($msg)
+    {
+        $this->logger->debug($msg);
+    }
+
+    /**
+     * Dump debug info.
+     *
+     * @deprecated since Monolog gather log, we don't need it
+     *
+     * @codeCoverageIgnore
+     */
+    protected function dump_dbg()
     {
-        $this->flags = $this->flags & ~$flag;
     }
 
     /**
@@ -796,7 +875,7 @@ class Readability implements LoggerAwareInterface
      * Prepare the HTML document for readability to scrape it.
      * This includes things like stripping javascript, CSS, and handling terrible markup.
      */
-    protected function prepDocument(): void
+    protected function prepDocument()
     {
         /*
          * In some cases a body element can't be found (if the HTML is totally hosed for example)
@@ -825,7 +904,7 @@ class Readability implements LoggerAwareInterface
      * Initialize a node with the readability object. Also checks the
      * className/id for special names to add to its score.
      */
-    protected function initializeNode(\DOMElement $node): void
+    protected function initializeNode(\DOMElement $node)
     {
         if (!isset($node->tagName)) {
             return;
@@ -861,7 +940,7 @@ class Readability implements LoggerAwareInterface
             case 'DD':
             case 'DT':
             case 'LI':
-                $readability->value -= 3;
+                $readability->value -= 2 * round($this->getLinkDensity($node), 0, \PHP_ROUND_HALF_UP);
                 break;
             case 'ASIDE':
             case 'FOOTER':
@@ -893,11 +972,11 @@ class Readability implements LoggerAwareInterface
      * Using a variety of metrics (content score, classname, element types), find the content that is
      * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
      *
-     * @param \DOMElement $page
+     * @param ?\DOMElement $page
      *
      * @return \DOMElement|false
      */
-    protected function grabArticle(\DOMElement $page = null)
+    protected function grabArticle($page = null)
     {
         if (!$page) {
             $page = $this->dom;
@@ -912,8 +991,7 @@ class Readability implements LoggerAwareInterface
 
         $allElements = $page->getElementsByTagName('*');
 
-        for ($nodeIndex = 0; $allElements->item($nodeIndex); ++$nodeIndex) {
+        for ($nodeIndex = 0; $node = $allElements->item($nodeIndex); ++$nodeIndex) {
-            $node = $allElements->item($nodeIndex);
             $tagName = $node->tagName;
 
             $nodeContent = $node->getInnerHTML();
@@ -922,35 +1000,14 @@ class Readability implements LoggerAwareInterface
                 continue;
             }
 
-            // Remove invisible nodes
-            if (!$this->isNodeVisible($node)) {
-                $this->logger->debug('Removing invisible node ' . $node->getNodePath());
-                $node->parentNode->removeChild($node);
-                --$nodeIndex;
-                continue;
-            }
-
-            // Remove unlikely candidates
-            $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
-
-            if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
-                preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
-                !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
-            ) {
-                $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '"');
-                $node->parentNode->removeChild($node);
-                --$nodeIndex;
-                continue;
-            }
-
             // Some well known site uses sections as paragraphs.
-            if (\in_array($tagName, $this->defaultTagsToScore, true)) {
+            if (0 === strcasecmp($tagName, 'p') || 0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'pre') || 0 === strcasecmp($tagName, 'section')) {
                 $nodesToScore[] = $node;
             }
 
             // Turn divs into P tags where they have been used inappropriately
             //  (as in, where they contain no other block level elements).
-            if ('div' === $tagName) {
+            if (0 === strcasecmp($tagName, 'div') || 0 === strcasecmp($tagName, 'article') || 0 === strcasecmp($tagName, 'section')) {
                 if (!preg_match($this->regexps['divToPElements'], $nodeContent)) {
                     $newNode = $this->dom->createElement('p');
 
@@ -965,46 +1022,27 @@ class Readability implements LoggerAwareInterface
                     }
                 } else {
                     // Will change these P elements back to text nodes after processing.
-                    $p = null;
+                    for ($i = 0, $il = $node->childNodes->length; $i < $il; ++$i) {
-                    // foreach does not handle removeChild very well
+                        $childNode = $node->childNodes->item($i);
-                    // See https://www.php.net/manual/en/domnode.removechild.php#90292
+
-                    $childs = iterator_to_array($node->childNodes);
+                        // it looks like sometimes the loop is going too far and we are retrieving a non-existant child
-                    foreach ($childs as $childNode) {
+                        if (null === $childNode) {
+                            continue;
+                        }
+
                         // executable tags (<?php or <?xml) warning
-                        if ($childNode instanceof \DOMProcessingInstruction) {
+                        if (\is_object($childNode) && 'DOMProcessingInstruction' === \get_class($childNode)) {
                             $childNode->parentNode->removeChild($childNode);
 
                             continue;
                         }
 
-                        if ($childNode instanceof \DOMText && '' === $this->getInnerText($childNode, true, true)) {
+                        if (\XML_TEXT_NODE === $childNode->nodeType) {
-                            /* $this->logger->debug('Remove empty text node'); */
-                            $childNode->parentNode->removeChild($childNode);
-
-                            continue;
-                        }
-
-                        if ($this->isPhrasingContent($childNode)) {
-                            if (null !== $p) {
-                                $p->appendChild($childNode);
-                            } elseif ('' !== $this->getInnerText($childNode, true, true)) {
                             $p = $this->dom->createElement('p');
+                            $p->setInnerHtml($childNode->nodeValue);
                             $p->setAttribute('data-readability-styled', 'true');
-                                $node->replaceChild($p, $childNode);
+                            $childNode->parentNode->replaceChild($p, $childNode);
-                                $p->appendChild($childNode);
                         }
-                        } elseif (null !== $p) {
-                            while ($p->lastChild && '' === $this->getInnerText($p->lastChild, true, true)) {
-                                $p->removeChild($p->lastChild);
-                            }
-                            $p = null;
-                        }
-                    }
-
-                    if ($this->hasSingleTagInsideElement($node, 'p') && $this->getLinkDensity($node) < 0.25) {
-                        $newNode = $node->childNodes->item(0);
-                        $node->parentNode->replaceChild($newNode, $node);
-                        $nodesToScore[] = $newNode;
                     }
                 }
             }
@@ -1018,13 +1056,14 @@ class Readability implements LoggerAwareInterface
          * Maybe eventually link density.
          */
         for ($pt = 0, $scored = \count($nodesToScore); $pt < $scored; ++$pt) {
-            $ancestors = $this->getAncestors($nodesToScore[$pt], 5);
+            $parentNode = $nodesToScore[$pt]->parentNode;
 
             // No parent node? Move on...
-            if (0 === \count($ancestors)) {
+            if (!$parentNode) {
                 continue;
             }
 
+            $grandParentNode = $parentNode->parentNode instanceof \DOMElement ? $parentNode->parentNode : null;
             $innerText = $this->getInnerText($nodesToScore[$pt]);
 
             // If this paragraph is less than MIN_PARAGRAPH_LENGTH (default:20) characters, don't even count it.
@@ -1032,6 +1071,17 @@ class Readability implements LoggerAwareInterface
                 continue;
             }
 
+            // Initialize readability data for the parent.
+            if (!$parentNode->hasAttribute('readability')) {
+                $this->initializeNode($parentNode);
+                $parentNode->setAttribute('data-candidate', 'true');
+            }
+
+            // Initialize readability data for the grandparent.
+            if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) {
+                $this->initializeNode($grandParentNode);
+                $grandParentNode->setAttribute('data-candidate', 'true');
+            }
             // Add a point for the paragraph itself as a base.
             $contentScore = 1;
             // Add points for any commas within this paragraph.
@@ -1039,26 +1089,25 @@ class Readability implements LoggerAwareInterface
             // For every SCORE_CHARS_IN_PARAGRAPH (default:100) characters in this paragraph, add another point. Up to 3 points.
             $contentScore += min(floor(mb_strlen($innerText) / self::SCORE_CHARS_IN_PARAGRAPH), 3);
             // For every SCORE_WORDS_IN_PARAGRAPH (default:20) words in this paragraph, add another point. Up to 3 points.
-            // $contentScore += min(floor($this->getWordCount($innerText) / self::SCORE_WORDS_IN_PARAGRAPH), 3);
+            $contentScore += min(floor($this->getWordCount($innerText) / self::SCORE_WORDS_IN_PARAGRAPH), 3);
-
+            /* TEST: For every positive/negative parent tag, add/substract half point. Up to 3 points. *\/
-            foreach ($ancestors as $level => $ancestor) {
+            $up = $nodesToScore[$pt];
-                if (!$ancestor->nodeName || !$ancestor->parentNode) {
+            $score = 0;
-                    return;
+            while ($up->parentNode instanceof \DOMElement) {
+                $up = $up->parentNode;
+                if (preg_match($this->regexps['positive'], $up->getAttribute('class') . ' ' . $up->getAttribute('id'))) {
+                    $score += 0.5;
+                } elseif (preg_match($this->regexps['negative'], $up->getAttribute('class') . ' ' . $up->getAttribute('id'))) {
+                    $score -= 0.5;
                 }
-
-                if (!$ancestor->hasAttribute('readability')) {
-                    $this->initializeNode($ancestor);
-                    $ancestor->setAttribute('data-candidate', 'true');
             }
+            $score = floor($score);
+            $contentScore += max(min($score, 3), -3);/**/
 
-                if (0 === $level) {
+            // Add the score to the parent. The grandparent gets half.
-                    $scoreDivider = 1;
+            $parentNode->getAttributeNode('readability')->value += $contentScore;
-                } elseif (1 === $level) {
+            if ($grandParentNode) {
-                    $scoreDivider = 2;
+                $grandParentNode->getAttributeNode('readability')->value += round($contentScore / self::GRANDPARENT_SCORE_DIVISOR);
-                } else {
-                    $scoreDivider = $level * 3;
-                }
-                $ancestor->getAttributeNode('readability')->value += $contentScore / $scoreDivider;
             }
         }
 
@@ -1082,6 +1131,18 @@ class Readability implements LoggerAwareInterface
 
             for ($c = $candidates->length - 1; $c >= 0; --$c) {
                 $node = $candidates->item($c);
+
+                // Remove unlikely candidates
+                $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
+
+                if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
+                    && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
+                    && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+                ) {
+                    $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
+                    $node->parentNode->removeChild($node);
+                    --$nodeIndex;
+                }
             }
             unset($candidates);
         }
@@ -1090,11 +1151,10 @@ class Readability implements LoggerAwareInterface
          * After we've calculated scores, loop through all of the possible candidate nodes we found
          * and find the one with the highest score.
          */
-        $topCandidates = array_fill(0, 5, null);
+        $topCandidate = null;
         if ($xpath) {
             // Using array of DOMElements after deletion is a path to DOOMElement.
             $candidates = $xpath->query('.//*[@data-candidate]', $page->documentElement);
-            $this->logger->debug('Candidates: ' . $candidates->length);
 
             for ($c = $candidates->length - 1; $c >= 0; --$c) {
                 $item = $candidates->item($c);
@@ -1105,25 +1165,14 @@ class Readability implements LoggerAwareInterface
                 $readability = $item->getAttributeNode('readability');
                 $readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, \PHP_ROUND_HALF_UP);
 
-                for ($t = 0; $t < 5; ++$t) {
+                if (!$topCandidate || $readability->value > (int) $topCandidate->getAttribute('readability')) {
-                    $aTopCandidate = $topCandidates[$t];
-
-                    if (!$aTopCandidate || $readability->value > (int) $aTopCandidate->getAttribute('readability')) {
                     $this->logger->debug('Candidate: ' . $item->getNodePath() . ' (' . $item->getAttribute('class') . ':' . $item->getAttribute('id') . ') with score ' . $readability->value);
-                        array_splice($topCandidates, $t, 0, [$item]);
+                    $topCandidate = $item;
-                        if (\count($topCandidates) > 5) {
-                            array_pop($topCandidates);
-                        }
-                        break;
-                    }
-                }
                 }
             }
 
-        $topCandidates = array_filter($topCandidates, function ($v, $idx) {
+            unset($candidates);
-            return 0 === $idx || null !== $v;
+        }
-        }, \ARRAY_FILTER_USE_BOTH);
-        $topCandidate = $topCandidates[0];
 
         /*
          * If we still have no top candidate, just use the body as a last resort.
@@ -1150,59 +1199,6 @@ class Readability implements LoggerAwareInterface
             }
 
             $this->initializeNode($topCandidate);
-        } elseif ($topCandidate) {
-            $alternativeCandidateAncestors = [];
-            foreach ($topCandidates as $candidate) {
-                if ((int) $candidate->getAttribute('readability') / (int) $topCandidate->getAttribute('readability') >= 0.75) {
-                    $ancestors = $this->getAncestors($candidate);
-                    $this->logger->debug('Adding ' . \count($ancestors) . ' alternative ancestors for ' . $candidate->getNodePath());
-                    $alternativeCandidateAncestors[] = $ancestors;
-                }
-            }
-            if (\count($alternativeCandidateAncestors) >= 3) {
-                $parentOfTopCandidate = $topCandidate->parentNode;
-                while ('body' !== $parentOfTopCandidate->nodeName) {
-                    $listsContainingThisAncestor = 0;
-                    for ($ancestorIndex = 0; $ancestorIndex < \count($alternativeCandidateAncestors) && $listsContainingThisAncestor < 3; ++$ancestorIndex) {
-                        $listsContainingThisAncestor += (int) \in_array($parentOfTopCandidate, $alternativeCandidateAncestors[$ancestorIndex], true);
-                    }
-                    if ($listsContainingThisAncestor >= 3) {
-                        $topCandidate = $parentOfTopCandidate;
-                        break;
-                    }
-                    $parentOfTopCandidate = $parentOfTopCandidate->parentNode;
-                }
-            }
-            if (!$topCandidate->hasAttribute('readability')) {
-                $this->initializeNode($topCandidate);
-            }
-            $parentOfTopCandidate = $topCandidate->parentNode;
-            $lastScore = (int) $topCandidate->getAttribute('readability');
-            $scoreThreshold = $lastScore / 3;
-            while ('body' !== $parentOfTopCandidate->nodeName) {
-                if (!$parentOfTopCandidate->hasAttribute('readability')) {
-                    $parentOfTopCandidate = $parentOfTopCandidate->parentNode;
-                    continue;
-                }
-                $parentScore = (int) $parentOfTopCandidate->getAttribute('readability');
-                if ($parentScore < $scoreThreshold) {
-                    break;
-                }
-                if ($parentScore > $lastScore) {
-                    $topCandidate = $parentOfTopCandidate;
-                    break;
-                }
-                $lastScore = (int) $parentOfTopCandidate->getAttribute('readability');
-                $parentOfTopCandidate = $parentOfTopCandidate->parentNode;
-            }
-            $parentOfTopCandidate = $topCandidate->parentNode;
-            while ('body' !== $parentOfTopCandidate->nodeName && 1 === $parentOfTopCandidate->childNodes->length) {
-                $topCandidate = $parentOfTopCandidate;
-                $parentOfTopCandidate = $topCandidate->parentNode;
-            }
-            if (!$topCandidate->hasAttribute('readability')) {
-                $this->initializeNode($topCandidate);
-            }
         }
 
         // Set table as the main node if resulted data is table element.
@@ -1228,10 +1224,9 @@ class Readability implements LoggerAwareInterface
         $articleContent = $this->dom->createElement('div');
         $articleContent->setAttribute('class', 'readability-content');
         $siblingScoreThreshold = max(10, ((int) $topCandidate->getAttribute('readability')) * 0.2);
-        $parentOfTopCandidate = $topCandidate->parentNode;
+        $siblingNodes = $topCandidate->parentNode->childNodes;
-        $siblingNodes = $parentOfTopCandidate->childNodes;
 
-        if (0 === $siblingNodes->length) {
+        if (null === $siblingNodes) {
             $siblingNodes = new \stdClass();
             $siblingNodes->length = 0;
         }
@@ -1244,7 +1239,8 @@ class Readability implements LoggerAwareInterface
 
             if ($siblingNode->isSameNode($topCandidate)) {
                 $append = true;
-            } else {
+            }
+
             $contentBonus = 0;
 
             // Give a bonus if sibling nodes and top candidates have the same classname.
@@ -1254,17 +1250,18 @@ class Readability implements LoggerAwareInterface
 
             if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
                 $append = true;
-                } elseif (0 === strcasecmp($siblingNodeName, 'p')) {
+            }
-                    $linkDensity = (int) $this->getLinkDensity($siblingNode);
+
+            if (0 === strcasecmp($siblingNodeName, 'p')) {
+                $linkDensity = $this->getLinkDensity($siblingNode);
                 $nodeContent = $this->getInnerText($siblingNode, true, true);
                 $nodeLength = mb_strlen($nodeContent);
 
                 if (($nodeLength > self::MIN_NODE_LENGTH && $linkDensity < self::MAX_LINK_DENSITY)
-                        || ($nodeLength < self::MIN_NODE_LENGTH && 0 === $nodeLength && 0 === $linkDensity && preg_match('/\.( |$)/', $nodeContent))) {
+                    || ($nodeLength < self::MIN_NODE_LENGTH && 0 === $linkDensity && preg_match('/\.( |$)/', $nodeContent))) {
                     $append = true;
                 }
             }
-            }
 
             if ($append) {
                 $this->logger->debug('Appending node: ' . $siblingNode->getNodePath());
@@ -1338,8 +1335,12 @@ class Readability implements LoggerAwareInterface
     /**
      * Get an element weight by attribute.
      * Uses regular expressions to tell if this element looks good or bad.
+     *
+     * @param string $attribute
+     *
+     * @return int
      */
-    protected function weightAttribute(\DOMElement $element, string $attribute): int
+    protected function weightAttribute(\DOMElement $element, $attribute)
     {
         if (!$element->hasAttribute($attribute)) {
             return 0;
@@ -1370,7 +1371,7 @@ class Readability implements LoggerAwareInterface
     /**
      * Will recreate previously deleted body property.
      */
-    protected function reinitBody(): void
+    protected function reinitBody()
     {
         if (!isset($this->body->childNodes)) {
             $this->body = $this->dom->createElement('body');
@@ -1382,15 +1383,20 @@ class Readability implements LoggerAwareInterface
      * Load HTML in a DOMDocument.
      * Apply Pre filters
      * Cleanup HTML using Tidy (or not).
+     *
+     * @todo This should be called in init() instead of from __construct
      */
-    private function loadHtml(): void
+    private function loadHtml()
     {
         $this->original_html = $this->html;
 
         $this->logger->debug('Parsing URL: ' . $this->url);
 
         if ($this->url) {
-            $this->domainRegExp = '/' . strtr((string) preg_replace('/www\d*\./', '', (string) parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
         }
 
         mb_internal_encoding('UTF-8');
@@ -1426,7 +1432,7 @@ class Readability implements LoggerAwareInterface
             unset($tidy);
         }
 
-        $this->html = '<meta charset="utf-8">' . (string) $this->html;
+        $this->html = self::entitizeNonAscii((string) $this->html);
 
         if ('html5lib' === $this->parser || 'html5' === $this->parser) {
             $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1442,58 +1448,23 @@ class Readability implements LoggerAwareInterface
             libxml_use_internal_errors(false);
         }
 
-        $this->dom->registerNodeClass(\DOMElement::class, \Readability\JSLikeHTMLElement::class);
+        $this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
-    }
-
-    private function getAncestors(\DOMElement $node, int $maxDepth = 0): array
-    {
-        $ancestors = [];
-        $i = 0;
-        while ($node->parentNode instanceof \DOMElement) {
-            $ancestors[] = $node->parentNode;
-            if (++$i === $maxDepth) {
-                break;
-            }
-            $node = $node->parentNode;
-        }
-
-        return $ancestors;
-    }
-
-    private function isPhrasingContent($node): bool
-    {
-        return \XML_TEXT_NODE === $node->nodeType
-            || \in_array(strtoupper($node->nodeName), $this->phrasingElements, true)
-            || (\in_array(strtoupper($node->nodeName), ['A', 'DEL', 'INS'], true) && !\in_array(false, array_map(function ($c) {
-                return $this->isPhrasingContent($c);
-            }, iterator_to_array($node->childNodes)), true));
-    }
-
-    private function hasSingleTagInsideElement(\DOMElement $node, string $tag): bool
-    {
-        if (1 !== $node->childNodes->length || $node->childNodes->item(0)->nodeName !== $tag) {
-            return false;
-        }
-
-        $a = array_filter(iterator_to_array($node->childNodes), function ($childNode) {
-            return $childNode instanceof \DOMText &&
-                preg_match($this->regexps['hasContent'], $this->getInnerText($childNode));
-        });
-
-        return 0 === \count($a);
     }
 
     /**
-     * Return whether a given node is visible or not.
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
      *
-     * Tidy must be configured to not clean the input for this function to
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
-     * work as expected, see $this->tidy_config['clean']
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     *
+     * @param string $html UTF-8 encoded document
      */
-    private function isNodeVisible(\DOMElement $node): bool
+    private static function entitizeNonAscii($html)
     {
-        return !($node->hasAttribute('style')
+        $convmap = [
-                    && preg_match($this->regexps['isNotVisible'], $node->getAttribute('style'))
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
-        )
+        ];
-                && !$node->hasAttribute('hidden');
+
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
     }
 }

tests/ReadabilityTest.php

@@ -4,138 +4,127 @@ namespace Tests\Readability;
 
 use Monolog\Handler\TestHandler;
 use Monolog\Logger;
-use Psr\Log\LoggerInterface;
-use Readability\JSLikeHTMLElement;
 use Readability\Readability;
 
 class ReadabilityTest extends \PHPUnit\Framework\TestCase
 {
-    /** @var TestHandler */
     public $logHandler;
-    /** @var LoggerInterface */
     public $logger;
 
     /**
      * @requires extension tidy
      */
-    public function testConstructDefault(): void
+    public function testConstructDefault()
     {
         $readability = $this->getReadability('');
-        $readability->init();
 
         $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
-    public function testConstructHtml5Parser(): void
+    public function testConstructHtml5Parser()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'html5lib');
-        $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
     }
 
     /**
      * @requires extension tidy
      */
-    public function testConstructSimple(): void
+    public function testConstructSimple()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0');
-        $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertTrue($readability->tidied);
     }
 
-    public function testConstructDefaultWithoutTidy(): void
+    public function testConstructDefaultWithoutTidy()
     {
         $readability = $this->getReadability('', null, 'libxml', false);
-        $readability->init();
 
         $this->assertNull($readability->url);
         $this->assertSame('', $readability->original_html);
         $this->assertFalse($readability->tidied);
 
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
-    public function testConstructSimpleWithoutTidy(): void
+    public function testConstructSimpleWithoutTidy()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'libxml', false);
-        $readability->init();
 
         $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
         $this->assertSame('<html/>', $readability->original_html);
         $this->assertFalse($readability->tidied);
     }
 
-    public function testInitNoContent(): void
+    public function testInitNoContent()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0');
         $res = $readability->init();
 
         $this->assertFalse($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
     }
 
-    public function testInitP(): void
+    public function testInitP()
     {
         $readability = $this->getReadability(str_repeat('<p>This is the awesome content :)</p>', 7), 'http://0.0.0.0');
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
-    public function testInitDivP(): void
+    public function testInitDivP()
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</div>', 'http://0.0.0.0');
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
-    public function testInitDiv(): void
+    public function testInitDiv()
     {
         $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithFootnotes(): void
+    public function testWithFootnotes()
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
@@ -143,16 +132,15 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertStringContainsString('readabilityLink-3', $readability->getContent()->getInnerHtml());
     }
 
-    public function testStandardClean(): void
+    public function testStandardClean()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
@@ -160,187 +148,178 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertStringNotContainsString('<h2>', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithIframe(): void
+    public function testWithIframe()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithArticle(): void
+    public function testWithArticle()
     {
         $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithAside(): void
+    public function testWithAside()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('<aside>', $readability->getContent()->getInnerHtml());
-        $this->assertStringContainsString('<footer readability="9"/>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithClasses(): void
+    public function testWithClasses()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithClassesWithoutLightClean(): void
+    public function testWithClassesWithoutLightClean()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithTd(): void
+    public function testWithTd()
     {
         $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithSameClasses(): void
+    public function testWithSameClasses()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithScript(): void
+    public function testWithScript()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testTitle(): void
+    public function testTitle()
     {
         $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertSame('this is my title', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testTitleWithDash(): void
+    public function testTitleWithDash()
     {
         $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertSame('title2 - title3', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testTitleWithDoubleDot(): void
+    public function testTitleWithDoubleDot()
     {
         $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertSame('title2 : title3', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testTitleTooShortUseH1(): void
+    public function testTitleTooShortUseH1()
     {
         $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertSame('this is my h1 title !', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
-    public function testAutoClosingIframeNotThrowingException(): void
-    {
-        $oldErrorReporting = error_reporting(\E_ALL | \E_STRICT);
-        $oldDisplayErrors = ini_set('display_errors', '1');
     // dummy function to be used to the next test
-        set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline, array $errcontext) {
+    public function error2Exception($code, $string, $file, $line)
-            throw new \Exception($errstr, $errno);
+    {
-        }, \E_ALL | \E_STRICT);
+        throw new \Exception($string, $code);
+    }
+
+    public function testAutoClosingIframeNotThrowingException()
+    {
+        $oldErrorReporting = error_reporting(\E_ALL);
+        $oldDisplayErrors = ini_set('display_errors', true);
+        set_error_handler([$this, 'error2Exception']);
 
         try {
             $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -365,17 +344,18 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
                       </div>
                     </div>
                   </div>
+                </div>
+              </div>
             </body>
             </html>';
 
             $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-            $readability->debug = true;
 
             $res = $readability->init();
 
             $this->assertTrue($res);
-            $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+            $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-            $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+            $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
             $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
             $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
         } finally {
@@ -390,7 +370,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     /**
      * This should generate an Exception "DOMElement::setAttribute(): ID post-60 already defined".
      */
-    public function testAppendIdAlreadyHere(): void
+    public function testAppendIdAlreadyHere()
     {
         $data = '<!DOCTYPE html>
             <html lang="fr">
@@ -437,16 +417,15 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             </html>';
 
         $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;
 
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        $this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
     }
 
-    public function testPostFilters(): void
+    public function testPostFilters()
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This <strong>is</strong> the awesome content :)</p>', 10) . '</div>', 'http://0.0.0.0');
         $readability->addPostFilter('!<strong[^>]*>(.*?)</strong>!is', '');
@@ -457,8 +436,10 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertStringContainsString('This  the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
-    public function testPreFilters(): void
+    public function testPreFilters()
     {
+        $this->markTestSkipped('Won\'t work until loadHtml() is moved in init() instead of __construct()');
+
         $readability = $this->getReadability('<div>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>', 'http://0.0.0.0');
         $readability->addPreFilter('!<b[^>]*>(.*?)</b>!is', '');
 
@@ -468,26 +449,24 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertStringContainsString('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
     }
 
-    public function testChildNodeGoneNull(): void
+    public function testChildNodeGoneNull()
     {
         // from http://www.ayyaantuu.net/ethiopia-targets-opposition-lawmakers/
-        $html = (string) file_get_contents('tests/fixtures/childNodeGoesNull.html');
+        $html = file_get_contents('tests/fixtures/childNodeGoesNull.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
     }
 
-    public function testKeepFootnotes(): void
+    public function testKeepFootnotes()
     {
         // from https://www.schreibdichte.de/blog/feed-aggregator-und-spaeter-lesen-dienst-im-team
-        $html = (string) file_get_contents('tests/fixtures/keepFootnotes.html');
+        $html = file_get_contents('tests/fixtures/keepFootnotes.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -495,62 +474,77 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertStringContainsString('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
     }
 
-    public function testWithWipedBody(): void
+    public function testWithWipedBody()
     {
         // from https://www.cs.cmu.edu/~rgs/alice-table.html
-        $html = (string) file_get_contents('tests/fixtures/wipedBody.html');
+        $html = file_get_contents('tests/fixtures/wipedBody.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
     }
 
-    public function dataForVisibleNode(): array
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle()
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang()
     {
         return [
-            'visible node' => [
+            'meta' => [
-                '<div>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
-                true,
+                'fr',
             ],
-            'display=none' => [
+            'head' => [
-                '<div style="display:none;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
                 false,
             ],
-            'display=inline' => [
+            'fragment' => [
-                '<div style="display:inline;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
-                true,
+                '',
-            ],
+                // tidy would add <html>.
-            'hidden attribute' => [
-                '<div hidden>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
                 false,
             ],
-            'missing display' => [
-                '<div style="color:#ccc;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
-                true,
-            ],
         ];
     }
 
     /**
-     * @dataProvider dataForVisibleNode
+     * @dataProvider dataForHtmlLang
      */
-    public function testVisibleNode(string $content, bool $shouldBeVisible): void
+    public function testHtmlLang($html, $lang, $useTidy = true)
     {
-        $readability = $this->getReadability($content, 'http://0.0.0.0');
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
-        $readability->debug = true;
         $res = $readability->init();
 
-        if ($shouldBeVisible) {
+        $this->assertTrue($res);
-            $this->assertStringContainsString('WONDERFUL content', $readability->getContent()->getInnerHtml());
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        } else {
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
-            $this->assertStringNotContainsString('WONDERFUL content', $readability->getContent()->getInnerHtml());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
-        }
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
     }
 
-    private function getReadability(string $html, string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
+    private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)
     {
         $readability = new Readability($html, $url, $parser, $useTidy);