Merge pull request #112 from j0k3r/fix/1.x/tidy-php85

Fix bad encoding for `tidy_repair_string`

.editorconfig

@@ -8,3 +8,6 @@ indent_style = space
 indent_size = 4
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[.github/**.yml]
+indent_size = 2

.gitattributes

@@ -6,4 +6,5 @@
 /.php_cs export-ignore
 /phpunit.xml.dist export-ignore
 /phpstan.neon export-ignore
+/.github export-ignore
 /tests export-ignore

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: j0k3r

.github/workflows/coding-standards.yml

@@ -0,0 +1,55 @@
+name: "CS"
+
+on:
+  pull_request:
+    branches:
+      - master
+      - 1.x
+  push:
+    branches:
+      - master
+      - 1.x
+
+env:
+  SYMFONY_PHPUNIT_VERSION: 7.5
+
+jobs:
+  coding-standards:
+    name: "CS Fixer & PHPStan"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "${{ matrix.php }}"
+          tools: cs2pr, composer:v2
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Add PHPStan"
+        run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Run PHP CS Fixer"
+        run: "php vendor/bin/php-cs-fixer fix --verbose --dry-run --format=checkstyle | cs2pr"
+
+      - name: "Install PHPUnit for PHPStan"
+        run: "php vendor/bin/simple-phpunit install"
+
+      - name: "Run PHPStan"
+        run: "php vendor/bin/phpstan analyse --error-format=checkstyle | cs2pr"

.github/workflows/continuous-integration.yml

@@ -0,0 +1,153 @@
+name: "CI"
+
+on:
+  pull_request:
+    branches:
+      - "1.x"
+  push:
+    branches:
+      - "1.x"
+
+env:
+  fail-fast: true
+
+jobs:
+  phpunit:
+    name: "PHPUnit (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.2"
+          - "7.3"
+          - "7.4"
+          - "8.0"
+          - "8.1"
+          - "8.2"
+          - "8.3"
+          - "8.4"
+          - "8.5"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"
+
+  phpunit-coverage:
+    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP with Xdebug"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "xdebug"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit (with coverage)"
+        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
+
+      - name: "Retrieve Coveralls phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+
+      - name: "Enable Coveralls phar"
+        run: "chmod +x php-coveralls.phar"
+
+      - name: "Upload to Coveralls"
+        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  phpunit-lowest:
+    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+          dependency-versions: "lowest"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"

.gitignore

@@ -3,3 +3,4 @@ coverage/
 composer.lock
 .php_cs.cache
 .phpunit.result.cache
+phpstan.neon

.php-cs-fixer.php

@@ -1,6 +1,11 @@
 <?php
 
-return PhpCsFixer\Config::create()
+$finder = (new PhpCsFixer\Finder())
+    ->in(__DIR__)
+    ->exclude(['vendor', 'var', 'web'])
+;
+
+return (new PhpCsFixer\Config())
     ->setUsingCache(true)
     ->setRiskyAllowed(true)
     ->setRules([
@@ -9,7 +14,7 @@ return PhpCsFixer\Config::create()
         'array_syntax' => ['syntax' => 'short'],
         'combine_consecutive_unsets' => true,
         'heredoc_to_nowdoc' => true,
-        'no_extra_consecutive_blank_lines' => ['break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block'],
+        'no_extra_blank_lines' => ['tokens' => ['break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block']],
         'no_unreachable_default_argument_value' => true,
         'no_useless_else' => true,
         'no_useless_return' => true,
@@ -21,10 +26,10 @@ return PhpCsFixer\Config::create()
         'strict_comparison' => true,
         'strict_param' => true,
         'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
+        // Pulled in by @Symfony with `const` but const visibility requires PHP ≥ 7.1
+        'visibility_required' => ['elements' => ['method', 'property']],
     ])
-    ->setFinder(
+    ->setFinder($finder)
-        PhpCsFixer\Finder::create()
-            ->exclude(['vendor'])
-            ->in(__DIR__)
-    )
 ;

.travis.yml

@@ -1,54 +0,0 @@
-dist: xenial
-os: linux
-language: php
-
-php:
-    - 5.6
-    - 7.0
-    - 7.1
-
-jobs:
-    include:
-        - php: 7.2
-          env: SYMFONY_PHPUNIT_VERSION=7.5
-        - php: 7.2
-          env: SYMFONY_PHPUNIT_VERSION=7.5 COMPOSER_FLAGS="--prefer-lowest"
-        - php: 7.3
-          env: SYMFONY_PHPUNIT_VERSION=7.5 CS_FIXER=run COVERAGE=run
-        - php: 7.4
-          env: SYMFONY_PHPUNIT_VERSION=7.5
-        - php: nightly
-          env: SYMFONY_PHPUNIT_VERSION=7.5
-    fast_finish: true
-    allow_failures:
-        - php: nightly
-
-cache:
-    directories:
-        - $HOME/.composer/cache/files
-
-before_install:
-    - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi;
-
-install:
-    # only install phpstan when we need it
-    - if [ "$CS_FIXER" = "run" ]; then composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest ; fi;
-    - composer update --prefer-dist --no-progress --no-suggest -o $COMPOSER_FLAGS
-    - php vendor/bin/simple-phpunit install
-
-before_script:
-    - mkdir -p build/logs
-    - if [ "$COVERAGE" = "run" ]; then PHPUNIT_FLAGS="--coverage-clover build/logs/clover.xml"; fi;
-
-script:
-    - php vendor/bin/simple-phpunit -v $PHPUNIT_FLAGS
-    - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/php-cs-fixer fix --verbose --dry-run ; fi;
-    - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/phpstan analyse src tests --no-progress --level 1 ; fi;
-
-after_success:
-    - |
-        if [ "$COVERAGE" = "run" ]; then
-            wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.2.0/php-coveralls.phar
-            chmod +x php-coveralls.phar
-            php php-coveralls.phar -v -x build/logs/clover.xml
-        fi

README.md

@@ -1,6 +1,6 @@
 # Readability
 
-[![Build Status](https://travis-ci.org/j0k3r/php-readability.svg?branch=master)](https://travis-ci.org/j0k3r/php-readability)
+![CI](https://github.com/j0k3r/php-readability/workflows/CI/badge.svg)
 [![Coverage Status](https://coveralls.io/repos/j0k3r/php-readability/badge.svg?branch=master&service=github)](https://coveralls.io/github/j0k3r/php-readability/?branch=master)
 [![Total Downloads](https://poser.pugx.org/j0k3r/php-readability/downloads)](https://packagist.org/packages/j0k3r/php-readability)
 [![License](https://poser.pugx.org/j0k3r/php-readability/license)](https://packagist.org/packages/j0k3r/php-readability)

composer.json

@@ -24,15 +24,15 @@
         "role": "Developer (original JS version)"
     }],
     "require": {
-        "php": ">=5.6.0",
+        "php": ">=7.2",
         "ext-mbstring": "*",
-        "psr/log": "^1.0",
+        "psr/log": "^1.0.1 || ^2.0 || ^3.0",
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24|^2.1",
+        "monolog/monolog": "^1.24|^2.1|^3.0",
-        "symfony/phpunit-bridge": "^4.2.3"
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -42,5 +42,10 @@
     },
     "autoload-dev": {
         "psr-4": { "Tests\\Readability\\": "tests/" }
+    },
+    "scripts": {
+        "fix": "php-cs-fixer fix --verbose --diff",
+        "phpstan": "phpstan analyze --memory-limit 512M",
+        "test": "simple-phpunit -v"
     }
 }

phpstan.dist.neon

@@ -1,8 +1,13 @@
+parameters:
+    level: 1
+    paths:
+        - src
+        - tests
+
+    # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
+    bootstrapFiles:
+        - vendor/bin/.phpunit/phpunit/vendor/autoload.php
+
 includes:
     - vendor/phpstan/phpstan-phpunit/extension.neon
     - vendor/phpstan/phpstan-phpunit/rules.neon
-
-parameters:
-    # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
-    autoload_files:
-        - vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php

src/JSLikeHTMLElement.php

@@ -39,23 +39,25 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for setting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
-     * @endcode
+     * ```
      */
     public function __set($name, $value)
     {
         if ('innerHTML' !== $name) {
             $trace = debug_backtrace();
-            trigger_error('Undefined property via __set(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], E_USER_NOTICE);
+            trigger_error('Undefined property via __set(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);
 
             return;
         }
 
         // first, empty the element
+        if (isset($this->childNodes)) {
             for ($x = $this->childNodes->length - 1; $x >= 0; --$x) {
                 $this->removeChild($this->childNodes->item($x));
             }
+        }
 
         // $value holds our new inner HTML
         $value = trim($value);
@@ -77,14 +79,13 @@ class JSLikeHTMLElement extends \DOMElement
         } else {
             // $value is probably ill-formed
             $f = new \DOMDocument();
-            $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
 
             // Using <htmlfragment> will generate a warning, but so will bad HTML
             // (and by this point, bad HTML is what we've got).
             // We use it (and suppress the warning) because an HTML fragment will
             // be wrapped around <html><body> tags which we don't really want to keep.
             // Note: despite the warning, if loadHTML succeeds it will return true.
-            $result = $f->loadHTML('<htmlfragment>' . $value . '</htmlfragment>');
+            $result = $f->loadHTML('<meta charset="utf-8"><htmlfragment>' . $value . '</htmlfragment>');
 
             if ($result) {
                 $import = $f->getElementsByTagName('htmlfragment')->item(0);
@@ -103,24 +104,26 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for getting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $string = $div->innerHTML;
-     * @endcode
+     * ```
      */
     public function __get($name)
     {
         if ('innerHTML' === $name) {
             $inner = '';
 
+            if (isset($this->childNodes)) {
                 foreach ($this->childNodes as $child) {
                     $inner .= $this->ownerDocument->saveXML($child);
                 }
+            }
 
             return $inner;
         }
 
         $trace = debug_backtrace();
-        trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], E_USER_NOTICE);
+        trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);
     }
 
     public function __toString()

src/Readability.php

@@ -116,10 +116,9 @@ class Readability implements LoggerAwareInterface
         'drop-empty-paras' => true,
         'drop-proprietary-attributes' => false,
         'enclose-text' => true,
-        'enclose-block-text' => true,
         'merge-divs' => true,
         // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
         'output-encoding' => 'utf8',
         'hide-comments' => true,
     ];
@@ -138,10 +137,6 @@ class Readability implements LoggerAwareInterface
     protected $useTidy;
     // raw HTML filters
     protected $pre_filters = [
-        // remove obvious scripts
-        '!<script[^>]*>(.*?)</script>!is' => '',
-        // remove obvious styles
-        '!<style[^>]*>(.*?)</style>!is' => '',
         // remove spans as we redefine styles and they're probably special-styled
         '!</?span[^>]*>!is' => '',
         // HACK: firewall-filtered content
@@ -158,7 +153,7 @@ class Readability implements LoggerAwareInterface
         // replace excessive br's
         '/<br\s*\/?>\s*<p/i' => '<p',
         // replace empty tags that break layouts
-        '!<(?:a|div|p)[^>]+/>!is' => '',
+        '!<(?:a|div|p|figure)[^>]+/>!is' => '',
         // remove all attributes on text tags
         // '!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
         // single newlines cleanup
@@ -188,7 +183,7 @@ class Readability implements LoggerAwareInterface
         $this->loadHtml();
     }
 
-    public function setLogger(LoggerInterface $logger)
+    public function setLogger(LoggerInterface $logger): void
     {
         $this->logger = $logger;
     }
@@ -292,6 +287,11 @@ class Readability implements LoggerAwareInterface
         $innerDiv->appendChild($articleContent);
         $overlay->appendChild($innerDiv);
 
+        // without tidy the body can (sometimes) be wiped, so re-create it
+        if (false === isset($this->body->childNodes)) {
+            $this->body = $this->dom->createElement('body');
+        }
+
         // Clear the old HTML, insert the new content.
         $this->body->setInnerHtml('');
         $this->body->appendChild($overlay);
@@ -336,9 +336,9 @@ class Readability implements LoggerAwareInterface
             $footnoteLink = $articleLink->cloneNode(true);
             $refLink = $this->dom->createElement('a');
             $footnote = $this->dom->createElement('li');
-            $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
+            $linkDomain = @parse_url($footnoteLink->getAttribute('href'), \PHP_URL_HOST);
             if (!$linkDomain && isset($this->url)) {
-                $linkDomain = @parse_url($this->url, PHP_URL_HOST);
+                $linkDomain = @parse_url($this->url, \PHP_URL_HOST);
             }
 
             $linkText = $this->getInnerText($articleLink);
@@ -363,7 +363,7 @@ class Readability implements LoggerAwareInterface
             $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
             $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
             $footnote->setInnerHtml('<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ');
-            $footnoteLink->setInnerHtml(('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText));
+            $footnoteLink->setInnerHtml('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText);
             $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
             $footnote->appendChild($footnoteLink);
 
@@ -390,6 +390,9 @@ class Readability implements LoggerAwareInterface
 
         $this->logger->debug($this->lightClean ? 'Light clean enabled.' : 'Standard clean enabled.');
 
+        $this->clean($articleContent, 'style');
+        $this->clean($articleContent, 'script');
+
         $this->cleanStyles($articleContent);
         $this->killBreaks($articleContent);
 
@@ -790,7 +793,7 @@ class Readability implements LoggerAwareInterface
      */
     public function addFlag($flag)
     {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
     }
 
     /**
@@ -800,13 +803,14 @@ class Readability implements LoggerAwareInterface
      */
     public function removeFlag($flag)
     {
-        $this->flags = $this->flags & ~$flag;
+        $this->flags &= ~$flag;
     }
 
     /**
      * Debug.
      *
      * @deprecated use $this->logger->debug() instead
+     *
      * @codeCoverageIgnore
      */
     protected function dbg($msg)
@@ -818,6 +822,7 @@ class Readability implements LoggerAwareInterface
      * Dump debug info.
      *
      * @deprecated since Monolog gather log, we don't need it
+     *
      * @codeCoverageIgnore
      */
     protected function dump_dbg()
@@ -935,7 +940,7 @@ class Readability implements LoggerAwareInterface
             case 'DD':
             case 'DT':
             case 'LI':
-                $readability->value -= 2 * round($this->getLinkDensity($node), 0, PHP_ROUND_HALF_UP);
+                $readability->value -= 2 * round($this->getLinkDensity($node), 0, \PHP_ROUND_HALF_UP);
                 break;
             case 'ASIDE':
             case 'FOOTER':
@@ -967,11 +972,11 @@ class Readability implements LoggerAwareInterface
      * Using a variety of metrics (content score, classname, element types), find the content that is
      * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
      *
-     * @param \DOMElement $page
+     * @param ?\DOMElement $page
      *
      * @return \DOMElement|false
      */
-    protected function grabArticle(\DOMElement $page = null)
+    protected function grabArticle($page = null)
     {
         if (!$page) {
             $page = $this->dom;
@@ -986,9 +991,15 @@ class Readability implements LoggerAwareInterface
 
         $allElements = $page->getElementsByTagName('*');
 
-        for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
+        for ($nodeIndex = 0; $node = $allElements->item($nodeIndex); ++$nodeIndex) {
             $tagName = $node->tagName;
 
+            $nodeContent = $node->getInnerHTML();
+            if (empty($nodeContent)) {
+                $this->logger->debug('Skipping empty node');
+                continue;
+            }
+
             // Some well known site uses sections as paragraphs.
             if (0 === strcasecmp($tagName, 'p') || 0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'pre') || 0 === strcasecmp($tagName, 'section')) {
                 $nodesToScore[] = $node;
@@ -997,11 +1008,11 @@ class Readability implements LoggerAwareInterface
             // Turn divs into P tags where they have been used inappropriately
             //  (as in, where they contain no other block level elements).
             if (0 === strcasecmp($tagName, 'div') || 0 === strcasecmp($tagName, 'article') || 0 === strcasecmp($tagName, 'section')) {
-                if (!preg_match($this->regexps['divToPElements'], $node->getInnerHTML())) {
+                if (!preg_match($this->regexps['divToPElements'], $nodeContent)) {
                     $newNode = $this->dom->createElement('p');
 
                     try {
-                        $newNode->setInnerHtml($node->getInnerHTML());
+                        $newNode->setInnerHtml($nodeContent);
 
                         $node->parentNode->replaceChild($newNode, $node);
                         --$nodeIndex;
@@ -1026,7 +1037,7 @@ class Readability implements LoggerAwareInterface
                             continue;
                         }
 
-                        if (XML_TEXT_NODE === $childNode->nodeType) {
+                        if (\XML_TEXT_NODE === $childNode->nodeType) {
                             $p = $this->dom->createElement('p');
                             $p->setInnerHtml($childNode->nodeValue);
                             $p->setAttribute('data-readability-styled', 'true');
@@ -1124,9 +1135,9 @@ class Readability implements LoggerAwareInterface
                 // Remove unlikely candidates
                 $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
 
-                if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
+                if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
-                    preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+                    && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
-                    !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+                    && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
                 ) {
                     $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
                     $node->parentNode->removeChild($node);
@@ -1152,7 +1163,7 @@ class Readability implements LoggerAwareInterface
                 // relatively small link density (5% or less) and be mostly unaffected by this operation.
                 // If not for this we would have used XPath to find maximum @readability.
                 $readability = $item->getAttributeNode('readability');
-                $readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, PHP_ROUND_HALF_UP);
+                $readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, \PHP_ROUND_HALF_UP);
 
                 if (!$topCandidate || $readability->value > (int) $topCandidate->getAttribute('readability')) {
                     $this->logger->debug('Candidate: ' . $item->getNodePath() . ' (' . $item->getAttribute('class') . ':' . $item->getAttribute('id') . ') with score ' . $readability->value);
@@ -1224,7 +1235,7 @@ class Readability implements LoggerAwareInterface
             $siblingNode = $siblingNodes->item($s);
             $siblingNodeName = $siblingNode->nodeName;
             $append = false;
-            $this->logger->debug('Looking at sibling node: ' . $siblingNode->getNodePath() . ((XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
+            $this->logger->debug('Looking at sibling node: ' . $siblingNode->getNodePath() . ((\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
 
             if ($siblingNode->isSameNode($topCandidate)) {
                 $append = true;
@@ -1233,11 +1244,11 @@ class Readability implements LoggerAwareInterface
             $contentBonus = 0;
 
             // Give a bonus if sibling nodes and top candidates have the same classname.
-            if (XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->getAttribute('class') === $topCandidate->getAttribute('class') && '' !== $topCandidate->getAttribute('class')) {
+            if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->getAttribute('class') === $topCandidate->getAttribute('class') && '' !== $topCandidate->getAttribute('class')) {
                 $contentBonus += ((int) $topCandidate->getAttribute('readability')) * 0.2;
             }
 
-            if (XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
+            if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
                 $append = true;
             }
 
@@ -1382,7 +1393,10 @@ class Readability implements LoggerAwareInterface
         $this->logger->debug('Parsing URL: ' . $this->url);
 
         if ($this->url) {
-            $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
         }
 
         mb_internal_encoding('UTF-8');
@@ -1418,7 +1432,7 @@ class Readability implements LoggerAwareInterface
             unset($tidy);
         }
 
-        $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');
+        $this->html = self::entitizeNonAscii((string) $this->html);
 
         if ('html5lib' === $this->parser || 'html5' === $this->parser) {
             $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1429,11 +1443,28 @@ class Readability implements LoggerAwareInterface
 
             $this->dom = new \DOMDocument();
             $this->dom->preserveWhiteSpace = false;
-            $this->dom->loadHTML($this->html, LIBXML_NOBLANKS | LIBXML_COMPACT | LIBXML_NOERROR);
+            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
 
             libxml_use_internal_errors(false);
         }
 
         $this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
     }
+
+    /**
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
+     *
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     *
+     * @param string $html UTF-8 encoded document
+     */
+    private static function entitizeNonAscii($html)
+    {
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];
+
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
+    }
 }

tests/ReadabilityTest.php

@@ -17,18 +17,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testConstructDefault()
     {
         $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);
 
         $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructHtml5Parser()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'html5lib');
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
     }
 
     /**
@@ -38,9 +39,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0');
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
         $this->assertTrue($readability->tidied);
     }
 
@@ -49,19 +50,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability = $this->getReadability('', null, 'libxml', false);
 
         $this->assertNull($readability->url);
-        $this->assertEquals('', $readability->original_html);
+        $this->assertSame('', $readability->original_html);
         $this->assertFalse($readability->tidied);
 
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructSimpleWithoutTidy()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'libxml', false);
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
         $this->assertFalse($readability->tidied);
     }
 
@@ -74,7 +75,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
     }
 
     public function testInitP()
@@ -85,9 +86,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
     public function testInitDivP()
@@ -98,244 +99,230 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
     public function testInitDiv()
     {
         $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithFootnotes()
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('readabilityFootnoteLink', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('readabilityFootnoteLink', $readability->getContent()->getInnerHtml());
-        $this->assertContains('readabilityLink-3', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('readabilityLink-3', $readability->getContent()->getInnerHtml());
     }
 
     public function testStandardClean()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('will NOT be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('will NOT be removed', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('<h2>', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('<h2>', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithIframe()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('nofollow', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithArticle()
     {
         $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('nofollow', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithAside()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('<aside>', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('<aside>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithClasses()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text should be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithClassesWithoutLightClean()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text should be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithTd()
     {
         $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithSameClasses()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithScript()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     public function testTitle()
     {
         $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my title', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('this is my title', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     public function testTitleWithDash()
     {
         $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 - title3', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('title2 - title3', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     public function testTitleWithDoubleDot()
     {
         $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 : title3', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('title2 : title3', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     public function testTitleTooShortUseH1()
     {
         $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my h1 title !', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('this is my h1 title !', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     // dummy function to be used to the next test
-    public function error2Exception($code, $string, $file, $line, $context)
+    public function error2Exception($code, $string, $file, $line)
     {
         throw new \Exception($string, $code);
     }
 
     public function testAutoClosingIframeNotThrowingException()
     {
-        error_reporting(E_ALL | E_STRICT);
+        $oldErrorReporting = error_reporting(\E_ALL);
-        ini_set('display_errors', true);
+        $oldDisplayErrors = ini_set('display_errors', true);
-        set_error_handler([$this, 'error2Exception'], E_ALL | E_STRICT);
+        set_error_handler([$this, 'error2Exception']);
 
+        try {
             $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
                 <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
 
@@ -358,19 +345,27 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
                       </div>
                     </div>
                   </div>
+                </div>
+              </div>
             </body>
             </html>';
 
             $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-        $readability->debug = true;
 
             $res = $readability->init();
 
             $this->assertTrue($res);
             $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
             $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
+            $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('3D Touch', $readability->getTitle()->getInnerHtml());
+            $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
+        } finally {
+            restore_error_handler();
+            if (false !== $oldDisplayErrors) {
+                ini_set('display_errors', $oldDisplayErrors);
+            }
+            error_reporting($oldErrorReporting);
+        }
     }
 
     /**
@@ -423,7 +418,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             </html>';
 
         $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;
 
         $res = $readability->init();
 
@@ -440,7 +434,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertContains('This  the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This  the awesome content :)', $readability->getContent()->getInnerHtml());
     }
 
     public function testPreFilters()
@@ -453,7 +447,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertContains('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
     }
 
     public function testChildNodeGoneNull()
@@ -462,7 +456,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = file_get_contents('tests/fixtures/childNodeGoesNull.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -475,12 +468,81 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = file_get_contents('tests/fixtures/keepFootnotes.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
-        $this->assertContains('<sup id="fnref1:fnfeed_2"><a href="#fn:fnfeed_2" class="footnote-ref">2</a></sup>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<sup id="fnref1:fnfeed_2"><a href="#fn:fnfeed_2" class="footnote-ref">2</a></sup>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
+    }
+
+    public function testWithWipedBody()
+    {
+        // from https://www.cs.cmu.edu/~rgs/alice-table.html
+        $html = file_get_contents('tests/fixtures/wipedBody.html');
+
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
+    }
+
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle()
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang()
+    {
+        return [
+            'meta' => [
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'head' => [
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
+                false,
+            ],
+            'fragment' => [
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
+                '',
+                // tidy would add <html>.
+                false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHtmlLang
+     */
+    public function testHtmlLang($html, $lang, $useTidy = true)
+    {
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
     }
 
     private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)

tests/fixtures/wipedBody.html

@@ -0,0 +1,67 @@
+
+<HTML>
+<HEAD>
+<TITLE>Alice's Adventures in Wonderland (Project Gutenberg)</TITLE>
+</HEAD>
+
+<frameset Rows="50, *">
+<frame src="alice-finfo.html">
+<frame src="alice-ftitle.html" name="alice-main">
+</frameset>
+
+<noframes>
+<BODY>
+<H1>Alice's Adventures in Wonderland</H1>
+                          <H1>Lewis Carroll</H1>
+               <H1>The Millennium Fulcrum Edition 3.0</H1>
+
+NOTE:  This is a hypertext formatted version of the Project Gutenberg edition.
+For more information, check the
+<A HREF="alice-small.txt">small print</A>
+or check out the
+<A HREF="ftp://uiarchive.cso.uiuc.edu/pub/etext/gutenberg/etext91/alice30.txt">
+full ascii text</A>.  The original Tenniel illustrations are also available
+due to the efforts of Project Gutenberg.  You can if you like, grab them as a
+<A HREF="ftp://uiarchive.cso.uiuc.edu/pub/etext/gutenberg/etext94/algif10.zip">
+"zip file"</A> or read the <A HREF="algif-small.txt">small print</A>
+that comes with them.
+This document is part of a small, but growing collection of html formatted
+etexts.  (Others may be found in either my <A
+HREF="http://www.cs.cmu.edu/Web/People/rgs/rgs-home.html">home page</A> or
+John Ockerbloom's indexes by <A
+HREF="http://www.cs.cmu.edu/Web/bookauthors.html">author</A> and <A
+HREF="http://www.cs.cmu.edu/Web/booktitles.html">title</A>.)
+I am still trying to figure out whether anyone else is interested in these
+on-line readable documents.  If you appreciate this document or would like to
+see more such, send me mail at "rgs@cs.cmu.edu".
+<P>
+<A HREF="alice01a.gif"><IMG SRC="alice01th.gif"></A>
+<P>
+<H2>CONTENTS</H2>
+<PRE>
+    CHAPTER I:    <A HREF="alice-I.html">Down the Rabbit-Hole</A>
+    CHAPTER II:   <A HREF="alice-II.html">The Pool of Tears</A>
+    CHAPTER III:  <A HREF="alice-III.html">A Caucus-Race and a Long Tale</A>
+    CHAPTER IV:   <A HREF="alice-IV.html">The Rabbit Sends in a Little Bill</A>
+    CHAPTER V:    <A HREF="alice-V.html">Advice from a Caterpillar</A>
+    CHAPTER VI:   <A HREF="alice-VI.html">Pig and Pepper</A>
+    CHAPTER VII:  <A HREF="alice-VII.html">A Mad Tea-Party</A>
+    CHAPTER VIII: <A HREF="alice-VIII.html">The Queen's Croquet-Ground</A>
+    CHAPTER IX:   <A HREF="alice-IX.html">The Mock Turtle's Story</A>
+    CHAPTER X:    <A HREF="alice-X.html">The Lobster Quadrille</A>
+    CHAPTER XI:   <A HREF="alice-XI.html">Who Stole the Tarts?</A>
+    CHAPTER XII:  <A HREF="alice-XII.html">Alice's Evidence</A>
+</PRE>
+
+<ADDRESS><A HREF="mailto:rgs@cs.cmu.edu">Robert Stockton</A></ADDRESS>
+<P>
+<!- Access counter added 5/25 1:49am ->
+<A href="http://www.dbasics.com/cgi-bin/pages.cgi?143205747"><IMG SRC="http://www.dbasics.com/cgi-bin/counter.cgi?143205747.2&(none)"></A> Access statistics from htmlZine
+<!- This page has been visited
+A HREF="http://counter.digits.com/wc?--info=yes&--name=rgsalice"
+IMG SRC="http://counter.digits.com/wc/-d/4/-r/-z/rgsalice"
+ ALIGN=absmiddle WIDTH=60 HEIGHT=20 BORDER=0 HSPACE=4 ALT="????"/A
+times since March 2, 1996. ->
+</BODY>
+</noframes>
+</HTML>