Merge pull request #52 from nicofrand/master

Skip empty (empty innerHTML) nodes when grabbing article
Merge pull request #62 from j0k3r/fix/avoid-wiped-body
2026-09-27 14:36:23 +00:00 · 2021-03-09 11:14:29 +01:00 · 2021-03-09 10:49:54 +01:00 · 2021-03-08 11:59:24 +01:00 · 2020-12-08 09:47:38 +01:00 · 2020-11-30 14:34:46 +01:00
16 changed files with 424 additions and 87 deletions
@@ -4,9 +4,10 @@ root = true
 ; Unix-style newlines
 [*]
 end_of_line = LF
-
-[*.php]
 indent_style = space
 indent_size = 4
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[.github/**.yml]
+indent_size = 2
@@ -4,7 +4,7 @@
 /.scrutinizer.yml export-ignore
 /.travis.yml export-ignore
 /.php_cs export-ignore
-/README.md export-ignore
 /phpunit.xml.dist export-ignore
-/tests export-ignore
 /phpstan.neon export-ignore
+/.github export-ignore
+/tests export-ignore
@@ -0,0 +1 @@
+github: j0k3r
@@ -0,0 +1,53 @@
+name: "CS"
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+env:
+  SYMFONY_PHPUNIT_VERSION: 7.5
+
+jobs:
+  coding-standards:
+    name: "CS Fixer & PHPStan"
+    runs-on: "ubuntu-18.04"
+
+    strategy:
+      matrix:
+        php:
+          - "7.3"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "${{ matrix.php }}"
+          tools: cs2pr, composer:v1
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Add PHPStan"
+        run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v1"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Run PHP CS Fixer"
+        run: "php vendor/bin/php-cs-fixer fix --verbose --dry-run --format=checkstyle | cs2pr"
+
+      - name: "Install PHPUnit for PHPStan"
+        run: "php vendor/bin/simple-phpunit install"
+
+      - name: "Run PHPStan"
+        run: "php vendor/bin/phpstan analyse --error-format=checkstyle | cs2pr"
@@ -0,0 +1,202 @@
+name: "CI"
+
+on:
+  pull_request:
+    branches:
+      - "master"
+  push:
+    branches:
+      - "master"
+
+env:
+  fail-fast: true
+
+jobs:
+  phpunit:
+    name: "PHPUnit (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-18.04"
+
+    strategy:
+      matrix:
+        php:
+          - "5.6"
+          - "7.0"
+          - "7.1"
+          - "7.2"
+          - "7.3"
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v1
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Force PHPUnit version"
+        if: matrix.php >= '7.2'
+        run: "echo $SYMFONY_PHPUNIT_VERSION"
+        env:
+          SYMFONY_PHPUNIT_VERSION: 7.5
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v1"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"
+
+  phpunit-coverage:
+    name: "PHPUnit coverage (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-18.04"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP with Xdebug"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "xdebug"
+          tools: composer:v1
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v1"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit (with coverage)"
+        run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
+        env:
+          SYMFONY_PHPUNIT_VERSION: 7.5
+
+      - name: "Retrieve Coveralls phar"
+        run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
+
+      - name: "Enable Coveralls phar"
+        run: "chmod +x php-coveralls.phar"
+
+      - name: "Upload to Coveralls"
+        run: "php php-coveralls.phar -v -x build/logs/clover.xml"
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  phpunit-lowest:
+    name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-18.04"
+
+    strategy:
+      matrix:
+        php:
+          - "7.2"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v1
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v1"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+          dependency-versions: "lowest"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"
+        env:
+          SYMFONY_PHPUNIT_VERSION: 7.5
+
+  phpunit-composerv2:
+    name: "PHPUnit with Composer v2 (PHP ${{ matrix.php }})"
+    runs-on: "ubuntu-18.04"
+
+    strategy:
+      matrix:
+        php:
+          - "7.4"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+        with:
+          fetch-depth: 2
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          php-version: "${{ matrix.php }}"
+          coverage: "none"
+          tools: composer:v2
+          extensions: tidy
+          ini-values: "date.timezone=Europe/Paris"
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Remove useless deps"
+        run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
+
+      - name: "Install dependencies with Composer"
+        uses: "ramsey/composer-install@v1"
+        with:
+          composer-options: "--optimize-autoloader --prefer-dist"
+
+      - name: "Setup logs"
+        run: "mkdir -p build/logs"
+
+      - name: "Run PHPUnit"
+        run: "php vendor/bin/simple-phpunit -v"
+        env:
+          SYMFONY_PHPUNIT_VERSION: 7.5
@@ -2,3 +2,4 @@ vendor/
 coverage/
 composer.lock
 .php_cs.cache
+.phpunit.result.cache
@@ -1,2 +1,12 @@
 tools:
    external_code_coverage: false
+
+build:
+    nodes:
+        analysis:
+            tests:
+                override:
+                    - php-scrutinizer-run
+            environment:
+                php:
+                    version: 7.2
@@ -1,42 +1,34 @@
+dist: xenial
+os: linux
 language: php

-php:
-    - 5.6
-    - 7.0
-    - 7.1
-    - 7.2
-    - 7.3
-    - nightly
+# avoid double Travis build when the PR is created on upstream
+if: |
+    type = pull_request OR \
+    branch = master

-matrix:
+jobs:
    include:
-        - php: 7.2
-          env: CS_FIXER=run
+        - php: 8.0
+          env: SYMFONY_PHPUNIT_VERSION=7.5
    fast_finish: true
    allow_failures:
-        - php: nightly
+        - php: 8.0

-# cache vendor dirs
 cache:
    directories:
-        - vendor
-        - $HOME/.composer/cache
+        - $HOME/.composer/cache/files

 before_install:
    - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi;

 install:
-    - composer self-update
+    - if [ "$CS_FIXER" != "run" ]; then composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update ; fi;
+    - composer update --prefer-dist --no-progress --no-suggest -o $
+    - php vendor/bin/simple-phpunit install

 before_script:
-    - if [ "$CS_FIXER" = "run" ]; then composer require phpstan/phpstan phpstan/phpstan-phpunit --dev -n ; fi;
-    - composer install -o --prefer-dist --no-interaction
+    - mkdir -p build/logs

 script:
-    - mkdir -p build/logs
-    - php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml
-    - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/php-cs-fixer fix --verbose --dry-run ; fi;
-    - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/phpstan analyse src tests --no-progress --level 1 ; fi;
-
-after_script:
-    - php vendor/bin/php-coveralls -v -x build/logs/clover.xml
+    - php vendor/bin/simple-phpunit -v
@@ -1,5 +1,6 @@
 # Readability

+![CI](https://github.com/j0k3r/php-readability/workflows/CI/badge.svg)
 [![Build Status](https://travis-ci.org/j0k3r/php-readability.svg?branch=master)](https://travis-ci.org/j0k3r/php-readability)
 [![Coverage Status](https://coveralls.io/repos/j0k3r/php-readability/badge.svg?branch=master&service=github)](https://coveralls.io/github/j0k3r/php-readability/?branch=master)
 [![Total Downloads](https://poser.pugx.org/j0k3r/php-readability/downloads)](https://packagist.org/packages/j0k3r/php-readability)
@@ -27,12 +27,11 @@
        "php": ">=5.6.0",
        "ext-mbstring": "*",
        "psr/log": "^1.0",
-        "electrolinux/php-html5lib": "^0.1.0"
+        "masterminds/html5": "^2.7"
    },
    "require-dev": {
-        "php-coveralls/php-coveralls": "^2.1",
        "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24",
+        "monolog/monolog": "^1.24|^2.1",
        "symfony/phpunit-bridge": "^4.2.3"
    },
    "suggest": {
@@ -1,8 +1,13 @@
+parameters:
+    level: 1
+    paths:
+        - src
+        - tests
+
+    # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
+    bootstrapFiles:
+        - vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php
+
 includes:
    - vendor/phpstan/phpstan-phpunit/extension.neon
    - vendor/phpstan/phpstan-phpunit/rules.neon
-
-parameters:
-    # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
-    autoload_files:
-        - vendor/bin/.phpunit/phpunit-7.4/vendor/autoload.php
@@ -26,6 +26,6 @@
    </filter>

    <!-- <logging>
-        <log type="coverage-html" target="coverage" title="Readability" charset="UTF-8" yui="true" highlight="true" lowUpperBound="35" highLowerBound="70"/>
+        <log type="coverage-html" target="coverage" lowUpperBound="35" highLowerBound="70"/>
    </logging> -->
 </phpunit>
@@ -47,15 +47,17 @@ class JSLikeHTMLElement extends \DOMElement
    {
        if ('innerHTML' !== $name) {
            $trace = debug_backtrace();
-            trigger_error('Undefined property via __set(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], E_USER_NOTICE);
+            trigger_error('Undefined property via __set(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);

            return;
        }

        // first, empty the element
+        if (isset($this->childNodes)) {
            for ($x = $this->childNodes->length - 1; $x >= 0; --$x) {
                $this->removeChild($this->childNodes->item($x));
            }
+        }

        // $value holds our new inner HTML
        $value = trim($value);
@@ -112,15 +114,17 @@ class JSLikeHTMLElement extends \DOMElement
        if ('innerHTML' === $name) {
            $inner = '';

+            if (isset($this->childNodes)) {
                foreach ($this->childNodes as $child) {
                    $inner .= $this->ownerDocument->saveXML($child);
                }
+            }

            return $inner;
        }

        $trace = debug_backtrace();
-        trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], E_USER_NOTICE);
+        trigger_error('Undefined property via __get(): ' . $name . ' in ' . $trace[0]['file'] . ' on line ' . $trace[0]['line'], \E_USER_NOTICE);
    }

    public function __toString()
@@ -2,7 +2,7 @@

 namespace Readability;

-use HTML5Lib\Parser;
+use Masterminds\HTML5;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
@@ -116,7 +116,6 @@ class Readability implements LoggerAwareInterface
        'drop-empty-paras' => true,
        'drop-proprietary-attributes' => false,
        'enclose-text' => true,
-        'enclose-block-text' => true,
        'merge-divs' => true,
        // 'merge-spans' => true,
        'input-encoding' => '????',
@@ -292,6 +291,11 @@ class Readability implements LoggerAwareInterface
        $innerDiv->appendChild($articleContent);
        $overlay->appendChild($innerDiv);

+        // without tidy the body can (sometimes) be wiped, so re-create it
+        if (false === isset($this->body->childNodes)) {
+            $this->body = $this->dom->createElement('body');
+        }
+
        // Clear the old HTML, insert the new content.
        $this->body->setInnerHtml('');
        $this->body->appendChild($overlay);
@@ -307,8 +311,6 @@ class Readability implements LoggerAwareInterface

    /**
     * Run any post-process modifications to article content as necessary.
-     *
-     * @param \DOMElement $articleContent
     */
    public function postProcessContent(\DOMElement $articleContent)
    {
@@ -321,8 +323,6 @@ class Readability implements LoggerAwareInterface
     * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
     *
     * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
-     *
-     * @param \DOMElement $articleContent
     */
    public function addFootnotes(\DOMElement $articleContent)
    {
@@ -340,9 +340,9 @@ class Readability implements LoggerAwareInterface
            $footnoteLink = $articleLink->cloneNode(true);
            $refLink = $this->dom->createElement('a');
            $footnote = $this->dom->createElement('li');
-            $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
+            $linkDomain = @parse_url($footnoteLink->getAttribute('href'), \PHP_URL_HOST);
            if (!$linkDomain && isset($this->url)) {
-                $linkDomain = @parse_url($this->url, PHP_URL_HOST);
+                $linkDomain = @parse_url($this->url, \PHP_URL_HOST);
            }

            $linkText = $this->getInnerText($articleLink);
@@ -385,8 +385,6 @@ class Readability implements LoggerAwareInterface
    /**
     * Prepare the article node for display. Clean out any inline styles,
     * iframes, forms, strip extraneous <p> tags, etc.
-     *
-     * @param \DOMNode $articleContent
     */
    public function prepArticle(\DOMNode $articleContent)
    {
@@ -558,7 +556,6 @@ class Readability implements LoggerAwareInterface
     * This is the amount of text that is inside a link divided by the total text in the node.
     * Can exclude external references to differentiate between simple text and menus/infoblocks.
     *
-     * @param \DOMElement $e
     * @param bool $excludeExternal
     *
     * @return int
@@ -586,8 +583,6 @@ class Readability implements LoggerAwareInterface
    /**
     * Get an element relative weight.
     *
-     * @param \DOMElement $e
-     *
     * @return int
     */
    public function getWeight(\DOMElement $e)
@@ -607,8 +602,6 @@ class Readability implements LoggerAwareInterface

    /**
     * Remove extraneous break tags from a node.
-     *
-     * @param \DOMElement $node
     */
    public function killBreaks(\DOMElement $node)
    {
@@ -623,12 +616,10 @@ class Readability implements LoggerAwareInterface
     *
     * Updated 2012-09-18 to preserve youtube/vimeo iframes
     *
-     * @param \DOMElement $e
     * @param string $tag
     */
    public function clean(\DOMElement $e, $tag)
    {
-        $currentItem = null;
        $targetList = $e->getElementsByTagName($tag);
        $isEmbed = ('audio' === $tag || 'video' === $tag || 'iframe' === $tag || 'object' === $tag || 'embed' === $tag);

@@ -659,7 +650,6 @@ class Readability implements LoggerAwareInterface
     * "Fishy" is an algorithm based on content length, classnames,
     * link density, number of images & embeds, etc.
     *
-     * @param \DOMElement $e
     * @param string $tag
     */
    public function cleanConditionally(\DOMElement $e, $tag)
@@ -670,7 +660,6 @@ class Readability implements LoggerAwareInterface

        $tagsList = $e->getElementsByTagName($tag);
        $curTagsLength = $tagsList->length;
-        $node = null;

        /*
         * Gather counts for other typical elements embedded within.
@@ -772,8 +761,6 @@ class Readability implements LoggerAwareInterface

    /**
     * Clean out spurious headers from an Element. Checks things like classnames and link density.
-     *
-     * @param \DOMElement $e
     */
    public function cleanHeaders(\DOMElement $e)
    {
@@ -915,8 +902,6 @@ class Readability implements LoggerAwareInterface
    /**
     * Initialize a node with the readability object. Also checks the
     * className/id for special names to add to its score.
-     *
-     * @param \DOMElement $node
     */
    protected function initializeNode(\DOMElement $node)
    {
@@ -954,7 +939,7 @@ class Readability implements LoggerAwareInterface
            case 'DD':
            case 'DT':
            case 'LI':
-                $readability->value -= 2 * round($this->getLinkDensity($node), 0, PHP_ROUND_HALF_UP);
+                $readability->value -= 2 * round($this->getLinkDensity($node), 0, \PHP_ROUND_HALF_UP);
                break;
            case 'ASIDE':
            case 'FOOTER':
@@ -1008,6 +993,12 @@ class Readability implements LoggerAwareInterface
        for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
            $tagName = $node->tagName;

+            $nodeContent = $node->getInnerHTML();
+            if (empty($nodeContent)) {
+                $this->logger->debug('Skipping empty node');
+                continue;
+            }
+
            // Some well known site uses sections as paragraphs.
            if (0 === strcasecmp($tagName, 'p') || 0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'pre') || 0 === strcasecmp($tagName, 'section')) {
                $nodesToScore[] = $node;
@@ -1016,11 +1007,11 @@ class Readability implements LoggerAwareInterface
            // Turn divs into P tags where they have been used inappropriately
            //  (as in, where they contain no other block level elements).
            if (0 === strcasecmp($tagName, 'div') || 0 === strcasecmp($tagName, 'article') || 0 === strcasecmp($tagName, 'section')) {
-                if (!preg_match($this->regexps['divToPElements'], $node->getInnerHTML())) {
+                if (!preg_match($this->regexps['divToPElements'], $nodeContent)) {
                    $newNode = $this->dom->createElement('p');

                    try {
-                        $newNode->setInnerHtml($node->getInnerHTML());
+                        $newNode->setInnerHtml($nodeContent);

                        $node->parentNode->replaceChild($newNode, $node);
                        --$nodeIndex;
@@ -1045,7 +1036,7 @@ class Readability implements LoggerAwareInterface
                            continue;
                        }

-                        if (XML_TEXT_NODE === $childNode->nodeType) {
+                        if (\XML_TEXT_NODE === $childNode->nodeType) {
                            $p = $this->dom->createElement('p');
                            $p->setInnerHtml($childNode->nodeValue);
                            $p->setAttribute('data-readability-styled', 'true');
@@ -1125,7 +1116,6 @@ class Readability implements LoggerAwareInterface
         */
        if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS) && $xpath) {
            $candidates = $xpath->query('.//*[(self::footer and count(//footer)<2) or (self::aside and count(//aside)<2)]', $page->documentElement);
-            $node = null;

            for ($c = $candidates->length - 1; $c >= 0; --$c) {
                $node = $candidates->item($c);
@@ -1137,7 +1127,6 @@ class Readability implements LoggerAwareInterface
            }

            $candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
-            $node = null;

            for ($c = $candidates->length - 1; $c >= 0; --$c) {
                $node = $candidates->item($c);
@@ -1173,7 +1162,7 @@ class Readability implements LoggerAwareInterface
                // relatively small link density (5% or less) and be mostly unaffected by this operation.
                // If not for this we would have used XPath to find maximum @readability.
                $readability = $item->getAttributeNode('readability');
-                $readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, PHP_ROUND_HALF_UP);
+                $readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, \PHP_ROUND_HALF_UP);

                if (!$topCandidate || $readability->value > (int) $topCandidate->getAttribute('readability')) {
                    $this->logger->debug('Candidate: ' . $item->getNodePath() . ' (' . $item->getAttribute('class') . ':' . $item->getAttribute('id') . ') with score ' . $readability->value);
@@ -1245,7 +1234,7 @@ class Readability implements LoggerAwareInterface
            $siblingNode = $siblingNodes->item($s);
            $siblingNodeName = $siblingNode->nodeName;
            $append = false;
-            $this->logger->debug('Looking at sibling node: ' . $siblingNode->getNodePath() . ((XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
+            $this->logger->debug('Looking at sibling node: ' . $siblingNode->getNodePath() . ((\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));

            if ($siblingNode->isSameNode($topCandidate)) {
                $append = true;
@@ -1254,11 +1243,11 @@ class Readability implements LoggerAwareInterface
            $contentBonus = 0;

            // Give a bonus if sibling nodes and top candidates have the same classname.
-            if (XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->getAttribute('class') === $topCandidate->getAttribute('class') && '' !== $topCandidate->getAttribute('class')) {
+            if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->getAttribute('class') === $topCandidate->getAttribute('class') && '' !== $topCandidate->getAttribute('class')) {
                $contentBonus += ((int) $topCandidate->getAttribute('readability')) * 0.2;
            }

-            if (XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
+            if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
                $append = true;
            }

@@ -1346,7 +1335,6 @@ class Readability implements LoggerAwareInterface
     * Get an element weight by attribute.
     * Uses regular expressions to tell if this element looks good or bad.
     *
-     * @param \DOMElement $element
     * @param string $attribute
     *
     * @return int
@@ -1404,7 +1392,7 @@ class Readability implements LoggerAwareInterface
        $this->logger->debug('Parsing URL: ' . $this->url);

        if ($this->url) {
-            $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, PHP_URL_HOST)), ['.' => '\.']) . '/';
+            $this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
        }

        mb_internal_encoding('UTF-8');
@@ -1442,8 +1430,8 @@ class Readability implements LoggerAwareInterface

        $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');

-        if ('html5lib' === $this->parser) {
-            $this->dom = Parser::parse($this->html);
+        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
+            $this->dom = (new HTML5())->loadHTML($this->html);
        }

        if ('libxml' === $this->parser) {
@@ -1451,7 +1439,7 @@ class Readability implements LoggerAwareInterface

            $this->dom = new \DOMDocument();
            $this->dom->preserveWhiteSpace = false;
-            $this->dom->loadHTML($this->html, LIBXML_NOBLANKS | LIBXML_COMPACT | LIBXML_NOERROR);
+            $this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);

            libxml_use_internal_errors(false);
        }
@@ -332,9 +332,9 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase

    public function testAutoClosingIframeNotThrowingException()
    {
-        error_reporting(E_ALL | E_STRICT);
+        error_reporting(\E_ALL | \E_STRICT);
        ini_set('display_errors', true);
-        set_error_handler([$this, 'error2Exception'], E_ALL | E_STRICT);
+        set_error_handler([$this, 'error2Exception'], \E_ALL | \E_STRICT);

        $data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
            <html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
@@ -483,6 +483,19 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
        $this->assertContains('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
    }

+    public function testWithWipedBody()
+    {
+        // from https://www.cs.cmu.edu/~rgs/alice-table.html
+        $html = file_get_contents('tests/fixtures/wipedBody.html');
+
+        $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
+        $readability->debug = true;
+        $res = $readability->init();
+
+        $this->assertTrue($res);
+        $this->assertContains('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
+    }
+
    private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)
    {
        $readability = new Readability($html, $url, $parser, $useTidy);
@@ -0,0 +1,67 @@
+
+<HTML>
+<HEAD>
+<TITLE>Alice's Adventures in Wonderland (Project Gutenberg)</TITLE>
+</HEAD>
+
+<frameset Rows="50, *">
+<frame src="alice-finfo.html">
+<frame src="alice-ftitle.html" name="alice-main">
+</frameset>
+
+<noframes>
+<BODY>
+<H1>Alice's Adventures in Wonderland</H1>
+                          <H1>Lewis Carroll</H1>
+               <H1>The Millennium Fulcrum Edition 3.0</H1>
+
+NOTE:  This is a hypertext formatted version of the Project Gutenberg edition.
+For more information, check the
+<A HREF="alice-small.txt">small print</A>
+or check out the
+<A HREF="ftp://uiarchive.cso.uiuc.edu/pub/etext/gutenberg/etext91/alice30.txt">
+full ascii text</A>.  The original Tenniel illustrations are also available
+due to the efforts of Project Gutenberg.  You can if you like, grab them as a
+<A HREF="ftp://uiarchive.cso.uiuc.edu/pub/etext/gutenberg/etext94/algif10.zip">
+"zip file"</A> or read the <A HREF="algif-small.txt">small print</A>
+that comes with them.
+This document is part of a small, but growing collection of html formatted
+etexts.  (Others may be found in either my <A
+HREF="http://www.cs.cmu.edu/Web/People/rgs/rgs-home.html">home page</A> or
+John Ockerbloom's indexes by <A
+HREF="http://www.cs.cmu.edu/Web/bookauthors.html">author</A> and <A
+HREF="http://www.cs.cmu.edu/Web/booktitles.html">title</A>.)
+I am still trying to figure out whether anyone else is interested in these
+on-line readable documents.  If you appreciate this document or would like to
+see more such, send me mail at "rgs@cs.cmu.edu".
+<P>
+<A HREF="alice01a.gif"><IMG SRC="alice01th.gif"></A>
+<P>
+<H2>CONTENTS</H2>
+<PRE>
+    CHAPTER I:    <A HREF="alice-I.html">Down the Rabbit-Hole</A>
+    CHAPTER II:   <A HREF="alice-II.html">The Pool of Tears</A>
+    CHAPTER III:  <A HREF="alice-III.html">A Caucus-Race and a Long Tale</A>
+    CHAPTER IV:   <A HREF="alice-IV.html">The Rabbit Sends in a Little Bill</A>
+    CHAPTER V:    <A HREF="alice-V.html">Advice from a Caterpillar</A>
+    CHAPTER VI:   <A HREF="alice-VI.html">Pig and Pepper</A>
+    CHAPTER VII:  <A HREF="alice-VII.html">A Mad Tea-Party</A>
+    CHAPTER VIII: <A HREF="alice-VIII.html">The Queen's Croquet-Ground</A>
+    CHAPTER IX:   <A HREF="alice-IX.html">The Mock Turtle's Story</A>
+    CHAPTER X:    <A HREF="alice-X.html">The Lobster Quadrille</A>
+    CHAPTER XI:   <A HREF="alice-XI.html">Who Stole the Tarts?</A>
+    CHAPTER XII:  <A HREF="alice-XII.html">Alice's Evidence</A>
+</PRE>
+
+<ADDRESS><A HREF="mailto:rgs@cs.cmu.edu">Robert Stockton</A></ADDRESS>
+<P>
+<!- Access counter added 5/25 1:49am ->
+<A href="http://www.dbasics.com/cgi-bin/pages.cgi?143205747"><IMG SRC="http://www.dbasics.com/cgi-bin/counter.cgi?143205747.2&(none)"></A> Access statistics from htmlZine
+<!- This page has been visited
+A HREF="http://counter.digits.com/wc?--info=yes&--name=rgsalice"
+IMG SRC="http://counter.digits.com/wc/-d/4/-r/-z/rgsalice"
+ ALIGN=absmiddle WIDTH=60 HEIGHT=20 BORDER=0 HSPACE=4 ALT="????"/A
+times since March 2, 1996. ->
+</BODY>
+</noframes>
+</HTML>
Author	SHA1	Message	Date
Jérémy BenoistandGitHub	9a490fac07	Merge pull request #52 from nicofrand/master Skip empty (empty innerHTML) nodes when grabbing article	2021-03-09 11:14:29 +01:00
Jérémy BenoistandGitHub	6f6b1f9e2b	Merge pull request #62 from j0k3r/fix/avoid-wiped-body Body can be wiped without tidy	2021-03-09 10:49:54 +01:00
Jeremy Benoist	ea1368fac0	Body can be wiped without tidy Re-create it in that case. Also run CS-Fixer.	2021-03-08 11:59:24 +01:00
Jérémy BenoistandGitHub	be81eb2a4f	Create FUNDING.yml	2020-12-08 09:47:38 +01:00
Jérémy BenoistandGitHub	9632c4df8c	Merge pull request #61 from j0k3r/github-actions Ditch Travis to use GitHub Actions	2020-11-30 14:34:46 +01:00
Jeremy Benoist	bd9ca1b2cd	Ditch Travis to use GitHub Actions	2020-11-30 14:18:40 +01:00
Jérémy BenoistandGitHub	6c917794a7	Merge pull request #60 from jtojnar/patch-1 readability: stop tidy from wrapping noscript text	2020-11-16 12:02:02 +01:00
Jan TojnarandGitHub	7cea79c23a	readability: stop tidy from wrapping noscript text HTML 4.01 Strict only allows block-level elements within noscript, form and blockquote. The `enclose-block-text` option fixes the instances when those elements contain inline elements or text by wrapping the children in paragraphs. HTML 5 has looser content model and allows noscript elements basically anywhere, including paragraphs, making the noscript elements inherit the parent element’s content model. This means that tidy will produce invalid HTML nesting paragraphs for `p > noscript > text`, a structure that would be invalid on two counts in HTML 4 Strict profile but is completely valid in HTML 5. Popular WordPress image lazy-loading code produces precisely that structure so tidy “corrects” it to invalid code. In a proper HTML parser, the produced code would force close the outer paragraph, making the noscript element its sibling instead of a child. The only reason this does not break Graby’s code for stripping the lazy-loading HTML is that libxml2 contains a bug counteracting this: https://gitlab.gnome.org/GNOME/libxml2/-/issues/205 Since all three elements allow flow content in HTML 5, it does not make much sense to enable this option any more. The only possible issues that could occur is producing HTML code not conforming to 4.01 Strict but that was never guaranteed, as our example shows, and having blockquotes contain text nodes not wrapped in paragraphs, which might be expected by some ancient stylesheets but that is only minor and easily fixable visual backwards incompatibility.	2020-11-14 22:13:53 +01:00
Jérémy BenoistandGitHub	c6425cc28b	Merge pull request #58 from j0k3r/fix/test-php8 Enable tests for PHP 8	2020-06-08 09:58:28 +02:00
Jeremy Benoist	e6ad806460	Enable tests on PHP 8	2020-06-08 09:24:50 +02:00
Jérémy BenoistandGitHub	3fa88461a0	Merge pull request #57 from j0k3r/fix/html5-parser Use a new deps for HTML5 parser	2020-06-08 07:59:42 +02:00
Jeremy Benoist	6a8ecf232f	Use a new deps for HTML5 parser `electrolinux/php-html5lib` was quite old and incompatible with the upcoming Composer 2.0. Jumping to `masterminds/html5` for the same result. Also the lib is maintained. Also: - keep README in vendors - use new Scrutinizer engine - test with lower deps - remove php-coveralls dev deps and download the phar during the CI build	2020-06-08 07:04:00 +02:00
Jérémy BenoistandGitHub	52b1ddba57	Merge pull request #55 from j0k3r/fix/cleanup-travis Cleanup travis	2020-04-12 16:38:09 +02:00
Jeremy Benoist	90c625bb57	Cleanup travis	2020-04-12 16:30:02 +02:00
Jérémy BenoistandGitHub	4d1c3b1777	Merge pull request #54 from j0k3r/fix/phpstan Fix PHPStan (again)	2020-01-06 14:26:41 +01:00
Jeremy Benoist	d649b59414	Fixing PHPUnit versions	2020-01-06 14:22:15 +01:00
Jeremy Benoist	44bebfc3d6	Only install PHPStan when we need it This is to avoid error when installing it on PHP < 7.1	2020-01-06 14:03:30 +01:00
Jeremy Benoist	b1acc9ed73	Fix PHPStan (again) Also cleanup	2019-11-19 14:09:29 +01:00
nicofrand	ff78c63e6d	Skip empty (empty innerHTML) nodes when grabbing article	2019-05-25 16:12:52 +02:00