Merge pull request #112 from j0k3r/fix/1.x/tidy-php85

Fix bad encoding for `tidy_repair_string`
Fix bad encoding for tidy_repair_string
2026-09-27 14:36:23 +00:00 · 2026-03-27 20:36:19 +01:00 · 2026-03-27 20:34:45 +01:00 · 2026-03-27 20:32:42 +01:00 · 2025-09-26 21:40:38 +02:00 · 2025-09-26 18:21:56 +02:00
11 changed files with 341 additions and 343 deletions
@@ -4,9 +4,11 @@ on:
  pull_request:
    branches:
      - master
+      - 1.x
  push:
    branches:
      - master
+      - 1.x

 env:
  SYMFONY_PHPUNIT_VERSION: 7.5
@@ -14,23 +16,23 @@ env:
 jobs:
  coding-standards:
    name: "CS Fixer & PHPStan"
-    runs-on: "ubuntu-18.04"
+    runs-on: "ubuntu-latest"

    strategy:
      matrix:
        php:
-          - "7.3"
+          - "7.4"

    steps:
      - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"

      - name: "Install PHP"
        uses: "shivammathur/setup-php@v2"
        with:
          coverage: "none"
          php-version: "${{ matrix.php }}"
-          tools: cs2pr, composer:v1
+          tools: cs2pr, composer:v2
          ini-values: "date.timezone=Europe/Paris"
        env:
          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -39,7 +41,7 @@ jobs:
        run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"

      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
        with:
          composer-options: "--optimize-autoloader --prefer-dist"

@@ -3,10 +3,10 @@ name: "CI"
 on:
  pull_request:
    branches:
-      - "master"
+      - "1.x"
  push:
    branches:
-      - "master"
+      - "1.x"

 env:
  fail-fast: true
@@ -14,163 +14,24 @@ env:
 jobs:
  phpunit:
    name: "PHPUnit (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-18.04"
+    runs-on: "ubuntu-latest"

    strategy:
      matrix:
        php:
-          - "5.6"
-          - "7.0"
-          - "7.1"
          - "7.2"
          - "7.3"
          - "7.4"
+          - "8.0"
+          - "8.1"
+          - "8.2"
+          - "8.3"
+          - "8.4"
+          - "8.5"

    steps:
      - name: "Checkout"
-        uses: "actions/checkout@v2"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "none"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Force PHPUnit version"
-        if: matrix.php >= '7.2'
-        run: "echo $SYMFONY_PHPUNIT_VERSION"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit"
-        run: "php vendor/bin/simple-phpunit -v"
-
-  phpunit-coverage:
-    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-18.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.4"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP with Xdebug"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "xdebug"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit (with coverage)"
-        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-      - name: "Retrieve Coveralls phar"
-        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
-
-      - name: "Enable Coveralls phar"
-        run: "chmod +x php-coveralls.phar"
-
-      - name: "Upload to Coveralls"
-        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
-        env:
-          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  phpunit-lowest:
-    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-18.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.2"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-        with:
-          fetch-depth: 2
-
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          php-version: "${{ matrix.php }}"
-          coverage: "none"
-          tools: composer:v1
-          extensions: tidy
-          ini-values: "date.timezone=Europe/Paris"
-        env:
-          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: "Remove useless deps"
-        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
-
-      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
-        with:
-          composer-options: "--optimize-autoloader --prefer-dist"
-          dependency-versions: "lowest"
-
-      - name: "Setup logs"
-        run: "mkdir -p build/logs"
-
-      - name: "Run PHPUnit"
-        run: "php vendor/bin/simple-phpunit -v"
-        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
-
-  phpunit-composerv2:
-    name: "PHPUnit with Composer v2 (PHP ${{ matrix.php }})"
-    runs-on: "ubuntu-18.04"
-
-    strategy:
-      matrix:
-        php:
-          - "7.4"
-
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v4"
        with:
          fetch-depth: 2

@@ -189,7 +50,7 @@ jobs:
        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"

      - name: "Install dependencies with Composer"
-        uses: "ramsey/composer-install@v1"
+        uses: "ramsey/composer-install@v3"
        with:
          composer-options: "--optimize-autoloader --prefer-dist"

@@ -198,5 +59,95 @@ jobs:

      - name: "Run PHPUnit"
        run: "php vendor/bin/simple-phpunit -v"
+
+  phpunit-coverage:
+    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP with Xdebug"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "xdebug"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
        env:
-          SYMFONY_PHPUNIT_VERSION: 7.5
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit (with coverage)"
+        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
+
+      - name: "Retrieve Coveralls phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+
+      - name: "Enable Coveralls phar"
+        run: "chmod +x php-coveralls.phar"
+
+      - name: "Upload to Coveralls"
+        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  phpunit-lowest:
+    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v3"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+          dependency-versions: "lowest"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"
@@ -3,3 +3,4 @@ coverage/
 composer.lock
 .php_cs.cache
 .phpunit.result.cache
+phpstan.neon
@@ -1,6 +1,11 @@
 <?php

-return PhpCsFixer\Config::create()
+$finder = (new PhpCsFixer\Finder())
+    ->in(__DIR__)
+    ->exclude(['vendor', 'var', 'web'])
+;
+
+return (new PhpCsFixer\Config())
    ->setUsingCache(true)
    ->setRiskyAllowed(true)
    ->setRules([
@@ -9,7 +14,7 @@ return PhpCsFixer\Config::create()
        'array_syntax' => ['syntax' => 'short'],
        'combine_consecutive_unsets' => true,
        'heredoc_to_nowdoc' => true,
-        'no_extra_consecutive_blank_lines' => ['break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block'],
+        'no_extra_blank_lines' => ['tokens' => ['break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block']],
        'no_unreachable_default_argument_value' => true,
        'no_useless_else' => true,
        'no_useless_return' => true,
@@ -21,10 +26,10 @@ return PhpCsFixer\Config::create()
        'strict_comparison' => true,
        'strict_param' => true,
        'concat_space' => ['spacing' => 'one'],
+        // Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
+        'no_null_property_initialization' => false,
+        // Pulled in by @Symfony with `const` but const visibility requires PHP ≥ 7.1
+        'visibility_required' => ['elements' => ['method', 'property']],
    ])
-    ->setFinder(
-        PhpCsFixer\Finder::create()
-            ->exclude(['vendor'])
-            ->in(__DIR__)
-    )
+    ->setFinder($finder)
 ;
@@ -1,34 +0,0 @@
-dist: xenial
-os: linux
-language: php
-
-# avoid double Travis build when the PR is created on upstream
-if: |
-    type = pull_request OR \
-    branch = master
-
-jobs:
-    include:
-        - php: 8.0
-          env: SYMFONY_PHPUNIT_VERSION=7.5
-    fast_finish: true
-    allow_failures:
-        - php: 8.0
-
-cache:
-    directories:
-        - $HOME/.composer/cache/files
-
-before_install:
-    - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi;
-
-install:
-    - if [ "$CS_FIXER" != "run" ]; then composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update ; fi;
-    - composer update --prefer-dist --no-progress --no-suggest -o $
-    - php vendor/bin/simple-phpunit install
-
-before_script:
-    - mkdir -p build/logs
-
-script:
-    - php vendor/bin/simple-phpunit -v
@@ -1,7 +1,6 @@
 # Readability

 ![CI](https://github.com/j0k3r/php-readability/workflows/CI/badge.svg)
-[![Build Status](https://travis-ci.org/j0k3r/php-readability.svg?branch=master)](https://travis-ci.org/j0k3r/php-readability)
 [![Coverage Status](https://coveralls.io/repos/j0k3r/php-readability/badge.svg?branch=master&service=github)](https://coveralls.io/github/j0k3r/php-readability/?branch=master)
 [![Total Downloads](https://poser.pugx.org/j0k3r/php-readability/downloads)](https://packagist.org/packages/j0k3r/php-readability)
 [![License](https://poser.pugx.org/j0k3r/php-readability/license)](https://packagist.org/packages/j0k3r/php-readability)
@@ -24,15 +24,15 @@
        "role": "Developer (original JS version)"
    }],
    "require": {
-        "php": ">=5.6.0",
+        "php": ">=7.2",
        "ext-mbstring": "*",
-        "psr/log": "^1.0",
+        "psr/log": "^1.0.1 || ^2.0 || ^3.0",
        "masterminds/html5": "^2.7"
    },
    "require-dev": {
        "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24|^2.1",
-        "symfony/phpunit-bridge": "^4.2.3"
+        "monolog/monolog": "^1.24|^2.1|^3.0",
+        "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0"
    },
    "suggest": {
        "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -42,5 +42,10 @@
    },
    "autoload-dev": {
        "psr-4": { "Tests\\Readability\\": "tests/" }
+    },
+    "scripts": {
+        "fix": "php-cs-fixer fix --verbose --diff",
+        "phpstan": "phpstan analyze --memory-limit 512M",
+        "test": "simple-phpunit -v"
    }
 }
@@ -6,7 +6,7 @@ parameters:

    # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
    bootstrapFiles:
-        - vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php
+        - vendor/bin/.phpunit/phpunit/vendor/autoload.php

 includes:
    - vendor/phpstan/phpstan-phpunit/extension.neon
@@ -39,9 +39,9 @@ class JSLikeHTMLElement extends \DOMElement
    /**
     * Used for setting innerHTML like it's done in JavaScript:.
     *
-     * @code
+     * ```php
     * $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
-     * @endcode
+     * ```
     */
    public function __set($name, $value)
    {
@@ -79,14 +79,13 @@ class JSLikeHTMLElement extends \DOMElement
        } else {
            // $value is probably ill-formed
            $f = new \DOMDocument();
-            $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');

            // Using <htmlfragment> will generate a warning, but so will bad HTML
            // (and by this point, bad HTML is what we've got).
            // We use it (and suppress the warning) because an HTML fragment will
            // be wrapped around <html><body> tags which we don't really want to keep.
            // Note: despite the warning, if loadHTML succeeds it will return true.
-            $result = $f->loadHTML('<htmlfragment>' . $value . '</htmlfragment>');
+            $result = $f->loadHTML('<meta charset="utf-8"><htmlfragment>' . $value . '</htmlfragment>');

            if ($result) {
                $import = $f->getElementsByTagName('htmlfragment')->item(0);
@@ -105,9 +104,9 @@ class JSLikeHTMLElement extends \DOMElement
    /**
     * Used for getting innerHTML like it's done in JavaScript:.
     *
-     * @code
+     * ```php
     * $string = $div->innerHTML;
-     * @endcode
+     * ```
     */
    public function __get($name)
    {
@@ -118,7 +118,7 @@ class Readability implements LoggerAwareInterface
        'enclose-text' => true,
        'merge-divs' => true,
        // 'merge-spans' => true,
-        'input-encoding' => '????',
+        'input-encoding' => 'utf8',
        'output-encoding' => 'utf8',
        'hide-comments' => true,
    ];
@@ -137,10 +137,6 @@ class Readability implements LoggerAwareInterface
    protected $useTidy;
    // raw HTML filters
    protected $pre_filters = [
-        // remove obvious scripts
-        '!<script[^>]*>(.*?)</script>!is' => '',
-        // remove obvious styles
-        '!<style[^>]*>(.*?)</style>!is' => '',
        // remove spans as we redefine styles and they're probably special-styled
        '!</?span[^>]*>!is' => '',
        // HACK: firewall-filtered content
@@ -148,7 +144,7 @@ class Readability implements LoggerAwareInterface
        // HACK: replace linebreaks plus br's with p's
        '!(<br[^>]*>[ \r\n\s]*){2,}!i' => '</p><p>',
        // replace noscripts
-        //'!</?noscript>!is' => '',
+        // '!</?noscript>!is' => '',
        // replace fonts to spans
        '!<(/?)font[^>]*>!is' => '<\\1span>',
    ];
@@ -157,10 +153,10 @@ class Readability implements LoggerAwareInterface
        // replace excessive br's
        '/<br\s*\/?>\s*<p/i' => '<p',
        // replace empty tags that break layouts
-        '!<(?:a|div|p)[^>]+/>!is' => '',
+        '!<(?:a|div|p|figure)[^>]+/>!is' => '',
        // remove all attributes on text tags
-        //'!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
-        //single newlines cleanup
+        // '!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
+        // single newlines cleanup
        "/\n+/" => "\n",
        // modern web...
        '!<pre[^>]*>\s*<code!is' => '<pre',
@@ -187,7 +183,7 @@ class Readability implements LoggerAwareInterface
        $this->loadHtml();
    }

-    public function setLogger(LoggerInterface $logger)
+    public function setLogger(LoggerInterface $logger): void
    {
        $this->logger = $logger;
    }
@@ -367,7 +363,7 @@ class Readability implements LoggerAwareInterface
            $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
            $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
            $footnote->setInnerHtml('<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ');
-            $footnoteLink->setInnerHtml(('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText));
+            $footnoteLink->setInnerHtml('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText);
            $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
            $footnote->appendChild($footnoteLink);

@@ -394,6 +390,9 @@ class Readability implements LoggerAwareInterface

        $this->logger->debug($this->lightClean ? 'Light clean enabled.' : 'Standard clean enabled.');

+        $this->clean($articleContent, 'style');
+        $this->clean($articleContent, 'script');
+
        $this->cleanStyles($articleContent);
        $this->killBreaks($articleContent);

@@ -794,7 +793,7 @@ class Readability implements LoggerAwareInterface
     */
    public function addFlag($flag)
    {
-        $this->flags = $this->flags | $flag;
+        $this->flags |= $flag;
    }

    /**
@@ -804,13 +803,14 @@ class Readability implements LoggerAwareInterface
     */
    public function removeFlag($flag)
    {
-        $this->flags = $this->flags & ~$flag;
+        $this->flags &= ~$flag;
    }

    /**
     * Debug.
     *
     * @deprecated use $this->logger->debug() instead
+     *
     * @codeCoverageIgnore
     */
    protected function dbg($msg)
@@ -822,6 +822,7 @@ class Readability implements LoggerAwareInterface
     * Dump debug info.
     *
     * @deprecated since Monolog gather log, we don't need it
+     *
     * @codeCoverageIgnore
     */
    protected function dump_dbg()
@@ -971,11 +972,11 @@ class Readability implements LoggerAwareInterface
     * Using a variety of metrics (content score, classname, element types), find the content that is
     * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
     *
-     * @param \DOMElement $page
+     * @param ?\DOMElement $page
     *
     * @return \DOMElement|false
     */
-    protected function grabArticle(\DOMElement $page = null)
+    protected function grabArticle($page = null)
    {
        if (!$page) {
            $page = $this->dom;
@@ -990,7 +991,7 @@ class Readability implements LoggerAwareInterface

        $allElements = $page->getElementsByTagName('*');

-        for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
+        for ($nodeIndex = 0; $node = $allElements->item($nodeIndex); ++$nodeIndex) {
            $tagName = $node->tagName;

            $nodeContent = $node->getInnerHTML();
@@ -1134,9 +1135,9 @@ class Readability implements LoggerAwareInterface
                // Remove unlikely candidates
                $unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');

-                if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
-                    preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
-                    !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
+                if (mb_strlen($unlikelyMatchString) > 3 // don't process "empty" strings
+                    && preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString)
+                    && !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
                ) {
                    $this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
                    $node->parentNode->removeChild($node);
@@ -1287,8 +1288,8 @@ class Readability implements LoggerAwareInterface

                // To ensure a node does not interfere with readability styles, remove its classnames & ids.
                // Now done via RegExp post_filter.
-                //$nodeToAppend->removeAttribute('class');
-                //$nodeToAppend->removeAttribute('id');
+                // $nodeToAppend->removeAttribute('class');
+                // $nodeToAppend->removeAttribute('id');
                // Append sibling and subtract from our list as appending removes a node.
                $articleContent->appendChild($nodeToAppend);
            }
@@ -1392,7 +1393,10 @@ class Readability implements LoggerAwareInterface
        $this->logger->debug('Parsing URL: ' . $this->url);

        if ($this->url) {
-            $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $host = parse_url($this->url, \PHP_URL_HOST);
+            if (null !== $host) {
+                $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', $host), ['.' => '\.']) . '/';
+            }
        }

        mb_internal_encoding('UTF-8');
@@ -1428,7 +1432,7 @@ class Readability implements LoggerAwareInterface
            unset($tidy);
        }

-        $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');
+        $this->html = self::entitizeNonAscii((string) $this->html);

        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
            $this->dom = (new HTML5())->loadHTML($this->html);
@@ -1446,4 +1450,21 @@ class Readability implements LoggerAwareInterface

        $this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
    }
+
+    /**
+     * Converts non-ASCII UTF-8 characters to numeric HTML entities.
+     *
+     * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+     * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+     *
+     * @param string $html UTF-8 encoded document
+     */
+    private static function entitizeNonAscii($html)
+    {
+        $convmap = [
+            0x80, 0x1FFFFF, 0, 0x10FFFF,
+        ];
+
+        return mb_encode_numericentity($html, $convmap, 'utf8', true);
+    }
 }
@@ -17,18 +17,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
    public function testConstructDefault()
    {
        $readability = $this->getReadability('');
+        $this->assertSame('utf8', $readability->tidy_config['input-encoding']);

        $this->assertNull($readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
    }

    public function testConstructHtml5Parser()
    {
        $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'html5lib');

-        $this->assertEquals('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('http://0.0.0.0', $readability->url);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame('<html/>', $readability->original_html);
    }

    /**
@@ -38,9 +39,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
    {
        $readability = $this->getReadability('<html/>', 'http://0.0.0.0');

-        $this->assertEquals('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('http://0.0.0.0', $readability->url);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame('<html/>', $readability->original_html);
        $this->assertTrue($readability->tidied);
    }

@@ -49,19 +50,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $readability = $this->getReadability('', null, 'libxml', false);

        $this->assertNull($readability->url);
-        $this->assertEquals('', $readability->original_html);
+        $this->assertSame('', $readability->original_html);
        $this->assertFalse($readability->tidied);

-        $this->assertInstanceOf('DomDocument', $readability->dom);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
    }

    public function testConstructSimpleWithoutTidy()
    {
        $readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'libxml', false);

-        $this->assertEquals('http://0.0.0.0', $readability->url);
-        $this->assertInstanceOf('DomDocument', $readability->dom);
-        $this->assertEquals('<html/>', $readability->original_html);
+        $this->assertSame('http://0.0.0.0', $readability->url);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame('<html/>', $readability->original_html);
        $this->assertFalse($readability->tidied);
    }

@@ -74,7 +75,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
    }

    public function testInitP()
@@ -85,9 +86,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
    }

    public function testInitDivP()
@@ -98,244 +99,230 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
    }

    public function testInitDiv()
    {
        $readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
    }

    public function testWithFootnotes()
    {
        $readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
-        $readability->debug = true;
        $readability->convertLinksToFootnotes = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('readabilityFootnoteLink', $readability->getContent()->getInnerHtml());
-        $this->assertContains('readabilityLink-3', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('readabilityFootnoteLink', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('readabilityLink-3', $readability->getContent()->getInnerHtml());
    }

    public function testStandardClean()
    {
        $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
-        $readability->debug = true;
        $readability->lightClean = false;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('will NOT be removed', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('<h2>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('will NOT be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('<h2>', $readability->getContent()->getInnerHtml());
    }

    public function testWithIframe()
    {
        $readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<div readability=', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('nofollow', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
    }

    public function testWithArticle()
    {
        $readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('nofollow', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
    }

    public function testWithAside()
    {
        $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('<aside>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('<aside>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
    }

    public function testWithClasses()
    {
        $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text should be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
    }

    public function testWithClassesWithoutLightClean()
    {
        $readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $readability->lightClean = false;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('alt="article"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text should be removed', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
    }

    public function testWithTd()
    {
        $readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
    }

    public function testWithSameClasses()
    {
        $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    public function testWithScript()
    {
        $readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
        $this->assertEmpty($readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    public function testTitle()
    {
        $readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my title', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertSame('this is my title', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    public function testTitleWithDash()
    {
        $readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 - title3', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertSame('title2 - title3', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    public function testTitleWithDoubleDot()
    {
        $readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('title2 : title3', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertSame('title2 : title3', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    public function testTitleTooShortUseH1()
    {
        $readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertEquals('this is my h1 title !', $readability->getTitle()->getInnerHtml());
-        $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
-        $this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
+        $this->assertSame('this is my h1 title !', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
+        $this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
    }

    // dummy function to be used to the next test
-    public function error2Exception($code, $string, $file, $line, $context)
+    public function error2Exception($code, $string, $file, $line)
    {
        throw new \Exception($string, $code);
    }

    public function testAutoClosingIframeNotThrowingException()
    {
-        error_reporting(\E_ALL | \E_STRICT);
-        ini_set('display_errors', true);
-        set_error_handler([$this, 'error2Exception'], \E_ALL | \E_STRICT);
+        $oldErrorReporting = error_reporting(\E_ALL);
+        $oldDisplayErrors = ini_set('display_errors', true);
+        set_error_handler([$this, 'error2Exception']);

+        try {
            $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
                <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">

@@ -358,19 +345,27 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
                      </div>
                    </div>
                  </div>
+                </div>
+              </div>
            </body>
            </html>';

            $readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
-        $readability->debug = true;

            $res = $readability->init();

            $this->assertTrue($res);
            $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
            $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
-        $this->assertContains('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('3D Touch', $readability->getTitle()->getInnerHtml());
+            $this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
+            $this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
+        } finally {
+            restore_error_handler();
+            if (false !== $oldDisplayErrors) {
+                ini_set('display_errors', $oldDisplayErrors);
+            }
+            error_reporting($oldErrorReporting);
+        }
    }

    /**
@@ -423,7 +418,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
            </html>';

        $readability = $this->getReadability($data, 'http://0.0.0.0');
-        $readability->debug = true;

        $res = $readability->init();

@@ -440,7 +434,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $res = $readability->init();

        $this->assertTrue($res);
-        $this->assertContains('This  the awesome content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This  the awesome content :)', $readability->getContent()->getInnerHtml());
    }

    public function testPreFilters()
@@ -453,7 +447,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $res = $readability->init();

        $this->assertTrue($res);
-        $this->assertContains('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
    }

    public function testChildNodeGoneNull()
@@ -462,7 +456,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $html = file_get_contents('tests/fixtures/childNodeGoesNull.html');

        $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
        $readability->convertLinksToFootnotes = true;
        $res = $readability->init();

@@ -475,12 +468,11 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $html = file_get_contents('tests/fixtures/keepFootnotes.html');

        $readability = $this->getReadability($html, 'http://0.0.0.0');
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
-        $this->assertContains('<sup id="fnref1:fnfeed_2"><a href="#fn:fnfeed_2" class="footnote-ref">2</a></sup>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<sup id="fnref1:fnfeed_2"><a href="#fn:fnfeed_2" class="footnote-ref">2</a></sup>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
    }

    public function testWithWipedBody()
@@ -489,11 +481,68 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $html = file_get_contents('tests/fixtures/wipedBody.html');

        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
-        $readability->debug = true;
        $res = $readability->init();

        $this->assertTrue($res);
-        $this->assertContains('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
+        $this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
+    }
+
+    // https://github.com/wallabag/wallabag/issues/8158
+    public function testCharsetAfterTitle()
+    {
+        $readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
+        $readability->convertLinksToFootnotes = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
+        $this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
+        $this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
+    }
+
+    /**
+     * @return array<string, array{0: string, 1: string, 2?: bool}>
+     */
+    public function dataForHtmlLang()
+    {
+        return [
+            'meta' => [
+                '<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'head' => [
+                '<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+            ],
+            'headless' => [
+                '<html lang="fr"><body><article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article></body></html>',
+                'fr',
+                // tidy would add <head> tag.
+                false,
+            ],
+            'fragment' => [
+                '<article>' . str_repeat('<p>Tous les êtres humains naissent libres et égaux en dignité et en droits. Ils sont doués de raison et de conscience et doivent agir les uns envers les autres dans un esprit de fraternité.</p>', 7) . '</article>',
+                '',
+                // tidy would add <html>.
+                false,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataForHtmlLang
+     */
+    public function testHtmlLang($html, $lang, $useTidy = true)
+    {
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
+        $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
+        $this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
+        $this->assertStringContainsString('êtres', $readability->getContent()->getInnerHtml());
    }

    private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)
Author	SHA1	Message	Date
Jérémy BenoistandGitHub	8ed91105ea	Merge pull request #112 from j0k3r/fix/1.x/tidy-php85 Fix bad encoding for `tidy_repair_string`	2026-03-27 20:36:19 +01:00
Jeremy Benoist	952588ee1f	Fix bad encoding for `tidy_repair_string` Tidy on PHP 8.5 is more restrictive on what can be given as input encoding. Before, it worked as an unknown value was converted to `utf8`.	2026-03-27 20:34:45 +01:00
Jeremy Benoist	e4e975db96	Add PHP 8.5 on CI	2026-03-27 20:32:42 +01:00
Jérémy BenoistandGitHub	774363e18d	Merge pull request #110 from j0k3r/fix/psr-monolog [1.x] Allow new psr/log & monolog	2025-09-26 21:40:38 +02:00
Jeremy Benoist	e6ca6f117f	Drop PHP < 7.2	2025-09-26 18:21:56 +02:00
Jeremy Benoist	0b21d4ab2d	Clean	2025-09-26 18:19:16 +02:00
Jeremy Benoist	720f0d5503	Allow new psr/log & monolog	2025-09-26 18:17:03 +02:00
Jérémy BenoistandGitHub	b9dde0f4cd	Merge pull request #107 from j0k3r/backport/encode [1.x] Backport character decoding regression	2025-06-03 10:02:58 +02:00
Jeremy Benoist	a21742b22a	Fix GA	2025-06-03 09:48:20 +02:00
Jan TojnarandJeremy Benoist	40219d4595	Fix character decoding regression when `title` precedes `meta[charset]` Because of PHP 8.2 deprecation, in `f14428e4c0`, we stopped converting non-ASCII characters to HTML entities. Instead, we started to explicitly insert `meta[charset]` tag at the start of the document. Later, we discovered that was breaking `html[lang]` so, in `efbbc86df9`, we made the insertion smarter. One of the improvements was that it would not insert the `meta[charset]` tag when it was already present. That, however, broke websites that had `title` tag before `meta[charset]`. On those, libxml2 would decode the `title` contents as ISO-8859-1. We could improve the logic (e.g. check that there is not text content before `meta[charset]`) or insert the tag unconditionally but it will probably be simplest to just go back to converting the non-ASCII characters to entities, just using non-deprecated function variant.	2025-06-03 09:48:20 +02:00
Jan TojnarandJeremy Benoist	7f304d03aa	tests: Check encoding was preserved in `testHtmlLang` The fix introduced in `efbbc86df9` alongside this test also manipulates `meta[charset]` but we were not checking if it does not break encoding.	2025-06-03 09:24:02 +02:00
Jérémy BenoistandGitHub	109a22662d	Merge pull request #103 from jtojnar/backports-local-no-domain [1.x] Backport parser_url + html[lang] fixes	2025-03-04 10:20:40 +01:00
Jan Tojnar	f1c6297e3c	Fix discarding `html[lang]` `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag. This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding. In `f14428e4c0`, we tried to resolve it by putting `meta[charset]` tag at the start of the HTML fragment. Unfortunately, it turns out that causes parser to auto-insert a `html` element, losing the attributes of the original `html` tag. Let’s try to insert the `meta[charset]` tag into the proper place in the HTML document. We do not need to use the same trick with `JSLikeHTMLElement::__set`. That expects smaller HTML fragments, not `html` documents, so creating `html` and `head` elements will not be a problem. (cherry picked from commit `efbbc86df9`) Had to strip type hints since we still target PHP 5.6.	2025-03-04 01:59:42 +01:00
Jan Tojnar	5afefcff34	tests: Remove pointless debug assignment It is unused since `8ab7d76cd5`. (cherry picked from commit `541fab34a0`)	2025-03-04 01:55:00 +01:00
Jan Tojnar	eb6ca1a99b	tests: Use ::class for DOMDocument class name Also capitalize it properly. (cherry picked from commit `90869d877e`)	2025-03-04 01:53:49 +01:00
Jan Tojnar	235baf965c	Do not set domainRegExp for local files `parse_url($this->url, \PHP_URL_HOST)` will return `null` for local filesystem path. Casting it to `string` will produce an empty regular expression, which would match any link when computing link density. (cherry picked from commit `c7208f6ad2`) This also fixes a warning since 1.x passes the `null` directly to `preg_replace` instead of explicitly casting it to `string`.	2025-03-03 23:35:40 +01:00
Jérémy BenoistandGitHub	487ce3a517	Merge pull request #98 from jtojnar/backports [1.x] Backport fixes	2025-02-24 09:51:12 +01:00
Jan Tojnar	0bae41248f	phpstan: Use standard config path This allows developer to create their own own config file, e.g. for setting `editorUrl`: https://phpstan.org/user-guide/output-format#opening-file-in-an-editor (cherry picked from commit `1d7cdf3a12`)	2025-02-24 09:47:42 +01:00
Jan Tojnar	ae0c20ab18	composer: Allow phpunit-bridge 6.0 and 7.0 This is required for PHP 8.4 support. On master this was done in `66215a6c80` and `1ac761d708`.	2025-02-24 09:47:42 +01:00
Jan Tojnar	84220bff99	ci: Add PHP 8.1 through 8.4 - 8.1 was introduced in `66215a6c80` - 8.2 and 8.3 was introduced in `9bdd3b6b2e` - 8.4 cherry picked from commit `5b9551d1e3`	2025-02-23 02:32:57 +01:00
Jan Tojnar	2ae758555a	Remove extra set_error_handler callback argument It is unused and would cause an error on PHP ≥ 8.0: https://www.php.net/manual/en/function.set-error-handler.php#refsect1-function.set-error-handler-parameters Not sure if the handler is even necessary – it was introduced in `175196d6c2` but I did not manage to reproduce the original error (Entity 'nbsp' not defined). It was probably fixed by `f2a43b476c`. (based on commit `da755013aa`)	2025-02-23 02:32:57 +01:00
Jan Tojnar	f5e25f3c9c	Avoid E_STRICT constant It will be deprecated in PHP 8.4 and it is meaningless nowadays anyway: https://wiki.php.net/rfc/deprecations_php_8_4#remove_e_strict_error_level_and_deprecate_e_strict_constant The use of the constant was introduced in `175196d6c2`. (cherry picked from commit `c7b10dcc45`)	2025-02-23 02:32:57 +01:00
Jan Tojnar	a209429e8b	tests: Fix “THE ERROR HANDLER HAS CHANGED!” (cherry picked from commit `23f824a1ce`)	2025-02-23 02:32:57 +01:00
Jan Tojnar	8421ed5962	Update coding style for upcoming PHP-CS-Fixer changes Once we bump minimum PHP version, we will get newer PHP-CS-Fixer, which will try to apply this cleanups. (partially cherry picked from commit `648d8c605b`) Though avoid disabling `modernize_strpos` since it was only introduced in PHP-CS-Fixer 3.2.0: https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/commit/2ca22a27c4ffb8cc46f20e7110b1dfcbb0c8c47c Also had to disable `visibility_required` for constants since those require PHP ≥ 7.1: https://cs.symfony.com/doc/rules/class_notation/visibility_required.html And remove type hint from `grabArticle` since implicitly nullable types were deprecated in PHP 8.4: https://wiki.php.net/rfc/deprecate-implicitly-nullable-types But we cannot use explicitly nullable types, which require PHP ≥ 7.1: https://wiki.php.net/rfc/nullable_types Also switch code blocks to Markdown syntax to work around `phpdoc_separation`, ApiGen uses Markdown these days anyway. (partially cherry picked from commit `9ed89bde92`)	2025-02-23 02:32:57 +01:00
Jan Tojnar	66864279fd	composer: Add scripts for development (cherry picked from commit `c5407ec07c`, except for rector)	2025-02-23 02:16:17 +01:00
Jan Tojnar	ae87c8ca8c	ci: Do not force PHPUnit version It should no longer be necessary and PHPUnit 7.5 is not compatible with PHP 8.4. master did that in `66215a6c80`.	2025-02-23 02:16:17 +01:00
Jan Tojnar	4adb0a96aa	PHPStan: Use stable PHPUnit path phpunit-bridge will create a symlink. (cherry picked from commit `2c6c6d5987`)	2025-02-23 02:16:17 +01:00
Jan Tojnar	baa0a78ab6	ci: Switch to composer v2 Composer v1 is not compatible with PHP 8.4. master switched in `66215a6c80`.	2025-02-23 01:34:11 +01:00
Jan Tojnar	2112dd95d1	ci: Update actions Mostly just of nodejs bump: - https://github.com/actions/checkout/releases/tag/v4.0.0 - https://github.com/ramsey/composer-install/releases/tag/3.0.0 (cherry picked from commit `7f4c6cfcbd`, resolving conflicts with previous bump in `82083c872b`)	2025-02-23 01:34:09 +01:00
Jan Tojnar	f4550ccc46	ci: Enable on 1.x branch	2025-02-23 01:34:05 +01:00
Jan Tojnar	6f4404030b	Do not use `mb_convert_encoding` with `HTML-ENTITIES` as target encoding This is deprecated since PHP 8.2: Deprecated: mb_convert_encoding(): Handling HTML entities via mbstring is deprecated; use htmlspecialchars, htmlentities, or mb_encode_numericentity/mb_decode_numericentity instead It was used because `DOMDocument`, which uses libxml2 internally, will parse the HTML as ISO-8859-1, unless the document contains an XML encoding declaration or HTML meta tag setting character set. Since first such element wins, putting the `meta[charset]` up front will ensure the parser uses the correct encoding, even if the document contains incorrect meta tag (e.g. when the document is converted to UTF-8 without also updating the metadata by the software passing it to Readability). https://stackoverflow.com/a/39148511/160386 (cherry picked from commit `f14428e4c0`)	2025-02-23 01:34:02 +01:00
Jérémy BenoistandGitHub	5638357306	Merge pull request #74 from Kdecherf/fix/pre-filters-failure	2022-06-13 06:15:24 +02:00
Kevin Decherf	651e8a6bb0	Strip script and style tags through ::clean() method instead of preg_replace Huge tags can lead to a failure of preg_replace, thus erasing the whole fetched content. Fixes https://github.com/wallabag/wallabag/issues/5847 Signed-off-by: Kevin Decherf <kevin@kdecherf.com>	2022-06-11 00:11:36 +02:00
Jérémy BenoistandGitHub	b1a20a9575	Merge pull request #68 from open-source-contributions/master Using assertSame to make assertion equal strict	2021-12-17 12:17:44 +01:00
peter279k	97c02e8ad4	Using assertSame to make assertion equal strict	2021-12-17 19:09:36 +08:00
Jérémy BenoistandGitHub	c506b7ebd7	Merge pull request #67 from j0k3r/fix/psr-log-void Fix deprecated message	2021-11-29 21:06:47 +01:00
Jeremy Benoist	d0af21814a	Ditch `assertContains` & `assertNotContains`	2021-11-29 21:04:56 +01:00
Jeremy Benoist	5b1eba79bd	Test on PHP 8 & drop Travis	2021-11-29 21:00:36 +01:00
Jérémy BenoistandGitHub	fabf096ce6	Fix deprecated message > Method "Psr\Log\LoggerAwareInterface::setLogger()" might add "void" as a native return type declaration in the future. Do the same in implementation "Readability\Readability" now to avoid errors or add an explicit @return annotation to suppress this message.	2021-11-29 20:50:10 +01:00
Jérémy BenoistandGitHub	8ce1663238	Merge pull request #66 from Kdecherf/fix/figure	2021-10-29 16:24:41 +02:00
Kevin Decherf	eb72a315c4	Clean empty figure tags without ending See 'Tag omission' https://developer.mozilla.org/en-US/docs/Web/HTML/Element/figure Signed-off-by: Kevin Decherf <kevin@kdecherf.com>	2021-10-29 15:36:19 +02:00
Jérémy BenoistandGitHub	d5330a9c28	Merge pull request #65 from j0k3r/fix/php-cs Fix PHPCS config file	2021-10-04 11:42:34 +02:00
Jeremy Benoist	635f8963dc	Fix Travis & PHPStan	2021-10-04 11:37:33 +02:00
Jeremy Benoist	19b2a25d96	Fix PHPCS config file	2021-10-04 11:32:34 +02:00