Merge pull request #112 from j0k3r/fix/1.x/tidy-php85

Fix bad encoding for `tidy_repair_string`

.github/workflows/coding-standards.yml

@@ -4,9 +4,11 @@ on:
   pull_request:
     branches:
       - master
+      - 1.x
   push:
     branches:
       - master
+      - 1.x
 
 env:
   SYMFONY_PHPUNIT_VERSION: 7.5
@@ -14,7 +16,7 @@ env:
 jobs:
   coding-standards:
     name: "CS Fixer & PHPStan"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
@@ -23,7 +25,7 @@ jobs:
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
 
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
@@ -39,7 +41,7 @@ jobs:
         run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 

.github/workflows/continuous-integration.yml

@@ -3,10 +3,10 @@ name: "CI"
 on:
   pull_request:
     branches:
-      - "master"
+      - "1.x"
   push:
     branches:
-      - "master"
+      - "1.x"
 
 env:
   fail-fast: true
@@ -14,164 +14,24 @@ env:
 jobs:
   phpunit:
     name: "PHPUnit (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
 
     strategy:
       matrix:
         php:
-          - "5.6"
-          - "7.0"
-          - "7.1"
           - "7.2"
           - "7.3"
           - "7.4"
           - "8.0"
+          - "8.1"
+          - "8.2"
+          - "8.3"
+          - "8.4"
+          - "8.5"
 
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "none"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Force PHPUnit version"
-        if: matrix.php >= '7.2'
-        run: "echo $SYMFONY_PHPUNIT_VERSION"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit"
-        run: "php vendor/bin/simple-phpunit -v"
-
-  phpunit-coverage:
-    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.4"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP with Xdebug"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "xdebug"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit (with coverage)"
-        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-      - name: "Retrieve Coveralls phar"
-        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
-
-      - name: "Enable Coveralls phar"
-        run: "chmod +x php-coveralls.phar"
-
-      - name: "Upload to Coveralls"
-        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
-        env:
-          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  phpunit-lowest:
-    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.2"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "none"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-          dependency-versions: "lowest"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit"
-        run: "php vendor/bin/simple-phpunit -v"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-  phpunit-composerv2:
-    name: "PHPUnit with Composer v2 (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-20.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.4"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
         with:
           fetch-depth: 2
 
@@ -190,7 +50,7 @@ jobs:
         run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
 
       - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
         with:
           composer-options: "--optimize-autoloader --prefer-dist"
 
@@ -199,5 +59,95 @@ jobs:
 
       - name: "Run PHPUnit"
         run: "php vendor/bin/simple-phpunit -v"
+
+  phpunit-coverage:
+    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP with Xdebug"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "xdebug"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
         env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit (with coverage)"
+        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
+
+      - name: "Retrieve Coveralls phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+
+      - name: "Enable Coveralls phar"
+        run: "chmod +x php-coveralls.phar"
+
+      - name: "Upload to Coveralls"
+        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  phpunit-lowest:
+    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+          dependency-versions: "lowest"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"

.gitignore

@@ -3,3 +3,4 @@ coverage/
 composer.lock
 .php_cs.cache
 .phpunit.result.cache
+phpstan.neon

.php-cs-fixer.php

@@ -26,6 +26,10 @@ return (new PhpCsFixer\Config())
         'strict_comparison' => true,
         'strict_param' => true,
         'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
+        // Pulled in by @Symfony with `const` but const visibility requires PHP ≥ 7.1
+        'visibility_required' => ['elements' => ['method', 'property']],
     ])
     ->setFinder($finder)
 ;

composer.json

@@ -24,15 +24,15 @@
         "role": "Developer (original JS version)"
     }],
     "require": {
-        "php": ">=5.6.0",
+        "php": ">=7.2",
         "ext-mbstring": "*",
-        "psr/log": "^1.0",
+        "psr/log": "^1.0.1 || ^2.0 || ^3.0",
         "masterminds/html5": "^2.7"
     },
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24|^2.1",
+        "monolog/monolog": "^1.24|^2.1|^3.0",
-        "symfony/phpunit-bridge": "^4.4|^5.3"
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0"
     },
     "suggest": {
         "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -42,5 +42,10 @@
     },
     "autoload-dev": {
         "psr-4": { "Tests\\Readability\\": "tests/" }
+    },
+    "scripts": {
+        "fix": "php-cs-fixer fix --verbose --diff",
+        "phpstan": "phpstan analyze --memory-limit 512M",
+        "test": "simple-phpunit -v"
     }
 }

phpstan.dist.neon

@@ -6,7 +6,7 @@ parameters:
 
     # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
     bootstrapFiles:
-        - vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php
+        - vendor/bin/.phpunit/phpunit/vendor/autoload.php
 
 includes:
     - vendor/phpstan/phpstan-phpunit/extension.neon

src/JSLikeHTMLElement.php

@@ -39,9 +39,9 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for setting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
-     * @endcode
+     * ```
      */
     public function __set($name, $value)
     {
@@ -79,14 +79,13 @@ class JSLikeHTMLElement extends \DOMElement
         } else {
             // $value is probably ill-formed
             $f = new \DOMDocument();
-            $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
 
             // Using <htmlfragment> will generate a warning, but so will bad HTML
             // (and by this point, bad HTML is what we've got).
             // We use it (and suppress the warning) because an HTML fragment will
             // be wrapped around <html><body> tags which we don't really want to keep.
             // Note: despite the warning, if loadHTML succeeds it will return true.
-            $result = $f->loadHTML('<htmlfragment>' . $value . '</htmlfragment>');
+            $result = $f->loadHTML('<meta charset="utf-8"><htmlfragment>' . $value . '</htmlfragment>');
 
             if ($result) {
                 $import = $f->getElementsByTagName('htmlfragment')->item(0);
@@ -105,9 +104,9 @@ class JSLikeHTMLElement extends \DOMElement
     /**
      * Used for getting innerHTML like it's done in JavaScript:.
      *
-     * @code
+     * ```php
      * $string = $div->innerHTML;
-     * @endcode
+     * ```
      */
     public function __get($name)
     {

src/Readability.php

@@ -118,7 +118,7 @@ class Readability implements LoggerAwareInterface
         'enclose-text' => true,
         'merge-divs' => true,
         // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
         'output-encoding' => 'utf8',
         'hide-comments' => true,
     ];
@@ -137,10 +137,6 @@ class Readability implements LoggerAwareInterface
     protected $useTidy;
     // raw HTML filters
     protected $pre_filters = [
-        // remove obvious scripts
-        '!<script[^>]*>(.*?)</script>!is' => '',
-        // remove obvious styles
-        '!<style[^>]*>(.*?)</style>!is' => '',
         // remove spans as we redefine styles and they're probably special-styled
         '!</?span[^>]*>!is' => '',
         // HACK: firewall-filtered content
@@ -187,10 +183,7 @@ class Readability implements LoggerAwareInterface
         $this->loadHtml();
     }
 
-    /**
+    public function setLogger(LoggerInterface $logger): void
-     * @return void
-     */
-    public function setLogger(LoggerInterface $logger)
     {
         $this->logger = $logger;
     }
@@ -370,7 +363,7 @@ class Readability implements LoggerAwareInterface
             $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
             $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
             $footnote->setInnerHtml('<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ');
-            $footnoteLink->setInnerHtml(('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText));
+            $footnoteLink->setInnerHtml('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText);
             $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
             $footnote->appendChild($footnoteLink);
 
@@ -397,6 +390,9 @@ class Readability implements LoggerAwareInterface
 
         $this->logger->debug($this->lightClean ? 'Light clean enabled.' : 'Standard clean enabled.');
 
+        $this->clean($articleContent, 'style');
+        $this->clean($articleContent, 'script');
+
         $this->cleanStyles($articleContent);
         $this->killBreaks($articleContent);
 
@@ -797,7 +793,7 @@ class Readability implements LoggerAwareInterface
      */
     public function addFlag($flag)
     {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
     }
 
     /**
@@ -807,13 +803,14 @@ class Readability implements LoggerAwareInterface
      */
     public function removeFlag($flag)
     {
-        $this->flags = $this->flags & ~$flag;
+        $this->flags &= ~$flag;
     }
 
     /**
      * Debug.
      *
      * @deprecated use $this->logger->debug() instead
+     *
      * @codeCoverageIgnore
      */
     protected function dbg($msg)
@@ -825,6 +822,7 @@ class Readability implements LoggerAwareInterface
      * Dump debug info.
      *
      * @deprecated since Monolog gather log, we don't need it
+     *
      * @codeCoverageIgnore
      */
     protected function dump_dbg()
@@ -974,11 +972,11 @@ class Readability implements LoggerAwareInterface
      * Using a variety of metrics (content score, classname, element types), find the content that is
      * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
      *
-     * @param \DOMElement $page
+     * @param ?\DOMElement $page
      *
      * @return \DOMElement|false
      */
-    protected function grabArticle(\DOMElement $page = null)
+    protected function grabArticle($page = null)
     {
         if (!$page) {
             $page = $this->dom;
@@ -993,7 +991,7 @@ class Readability implements LoggerAwareInterface
 
         $allElements = $page->getElementsByTagName('*');
 
-        for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
+        for ($nodeIndex = 0; $node = $allElements->item($nodeIndex); ++$nodeIndex) {
             $tagName = $node->tagName;
 
             $nodeContent = $node->getInnerHTML();
@@ -1137,9 +1135,9 @@ class Readability implements LoggerAwareInterface
                 // Remove unlikely candidates
                 $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
 
-                if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
+                if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
-                    preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+                    && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
-                    !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+                    && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
                 ) {
                     $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
                     $node->parentNode->removeChild($node);
@@ -1395,7 +1393,10 @@ class Readability implements LoggerAwareInterface
         $this->logger->debug('Parsing URL: ' . $this->url);
 
         if ($this->url) {
-            $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
         }
 
         mb_internal_encoding('UTF-8');
@@ -1431,7 +1432,7 @@ class Readability implements LoggerAwareInterface
             unset($tidy);
         }
 
-        $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');
+        $this->html = self::entitizeNonAscii((string) $this->html);
 
         if ('html5lib' === $this->parser || 'html5' === $this->parser) {
             $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1449,4 +1450,21 @@ class Readability implements LoggerAwareInterface
 
         $this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
     }
+
+    /**
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
+     *
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     *
+     * @param string $html UTF-8 encoded document
+     */
+    private static function entitizeNonAscii($html)
+    {
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];
+
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
+    }
 }

tests/ReadabilityTest.php

@@ -17,18 +17,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testConstructDefault()
     {
         $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);
 
         $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructHtml5Parser()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'html5lib');
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
     }
 
     /**
@@ -38,9 +39,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0');
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
         $this->assertTrue($readability->tidied);
     }
 
@@ -49,19 +50,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $readability = $this->getReadability('', null, 'libxml', false);
 
         $this->assertNull($readability->url);
-        $this->assertEquals('', $readability->original_html);
+        $this->assertSame('', $readability->original_html);
         $this->assertFalse($readability->tidied);
 
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
     }
 
     public function testConstructSimpleWithoutTidy()
     {
         $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'libxml', false);
 
-        $this->assertEquals('http://0.0.0.0', $readability->url);
+        $this->assertSame('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('<html/>', $readability->original_html);
         $this->assertFalse($readability->tidied);
     }
 
@@ -106,7 +107,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testInitDiv()
     {
         $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -120,7 +120,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithFootnotes()
     {
         $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -137,7 +136,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testStandardClean()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -154,7 +152,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithIframe()
     {
         $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -169,7 +166,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithArticle()
     {
         $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -184,7 +180,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithAside()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -199,7 +194,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClasses()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -214,7 +208,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithClassesWithoutLightClean()
     {
         $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->lightClean = false;
         $res = $readability->init();
 
@@ -230,7 +223,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithTd()
     {
         $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -243,7 +235,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithSameClasses()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -257,7 +248,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testWithScript()
     {
         $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -271,13 +261,12 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitle()
     {
         $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my title', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('this is my title', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
@@ -285,13 +274,12 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDash()
     {
         $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 - title3', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('title2 - title3', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
@@ -299,13 +287,12 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleWithDoubleDot()
     {
         $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 : title3', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('title2 : title3', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
@@ -313,29 +300,29 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
     public function testTitleTooShortUseH1()
     {
         $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
         $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my h1 title !', $readability->getTitle()->getInnerHtml());
+        $this->assertSame('this is my h1 title !', $readability->getTitle()->getInnerHtml());
         $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
     }
 
     // dummy function to be used to the next test
-    public function error2Exception($code, $string, $file, $line, $context)
+    public function error2Exception($code, $string, $file, $line)
     {
         throw new \Exception($string, $code);
     }
 
     public function testAutoClosingIframeNotThrowingException()
     {
-        error_reporting(\E_ALL | \E_STRICT);
+        $oldErrorReporting = error_reporting(\E_ALL);
-        ini_set('display_errors', true);
+        $oldDisplayErrors = ini_set('display_errors', true);
-        set_error_handler([$this, 'error2Exception'], \E_ALL | \E_STRICT);
+        set_error_handler([$this, 'error2Exception']);
 
+        try {
             $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
                 <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
 
@@ -358,11 +345,12 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
                       </div>
                     </div>
                   </div>
+                </div>
+              </div>
             </body>
             </html>';
 
             $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-        $readability->debug = true;
 
             $res = $readability->init();
 
@@ -371,6 +359,13 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
             $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
             $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
+        } finally {
+            restore_error_handler();
+            if (false !== $oldDisplayErrors) {
+                ini_set('display_errors', $oldDisplayErrors);
+            }
+            error_reporting($oldErrorReporting);
+        }
     }
 
     /**
@@ -423,7 +418,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
             </html>';
 
         $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;
 
         $res = $readability->init();
 
@@ -462,7 +456,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = file_get_contents('tests/fixtures/childNodeGoesNull.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $readability->convertLinksToFootnotes = true;
         $res = $readability->init();
 
@@ -475,7 +468,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = file_get_contents('tests/fixtures/keepFootnotes.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
@@ -489,13 +481,70 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $html = file_get_contents('tests/fixtures/wipedBody.html');
 
         $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
-        $readability->debug = true;
         $res = $readability->init();
 
         $this->assertTrue($res);
         $this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
     }
 
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle()
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang()
+    {
+        return [
+            'meta' => [
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'head' => [
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
+                false,
+            ],
+            'fragment' => [
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
+                '',
+                // tidy would add <html>.
+                false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHtmlLang
+     */
+    public function testHtmlLang($html, $lang, $useTidy = true)
+    {
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
+    }
+
     private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)
     {
         $readability = new Readability($html, $url, $parser, $useTidy);