Merge pull request #57 from j0k3r/fix/html5-parser

Use a new deps for HTML5 parser

.editorconfig

@@ -4,8 +4,6 @@ root = true
 ; Unix-style newlines
 [*]
 end_of_line = LF
-
-[*.php]
 indent_style = space
 indent_size = 4
 trim_trailing_whitespace = true

.gitattributes

@@ -4,7 +4,6 @@
 /.scrutinizer.yml export-ignore
 /.travis.yml export-ignore
 /.php_cs export-ignore
-/README.md export-ignore
 /phpunit.xml.dist export-ignore
-/tests export-ignore
 /phpstan.neon export-ignore
+/tests export-ignore

.gitignore

@@ -2,3 +2,4 @@ vendor/
 coverage/
 composer.lock
 .php_cs.cache
+.phpunit.result.cache

.scrutinizer.yml

@@ -1,2 +1,12 @@
 tools:
     external_code_coverage: false
+
+build:
+    nodes:
+        analysis:
+            tests:
+                override:
+                    - php-scrutinizer-run
+            environment:
+                php:
+                    version: 7.2

.travis.yml

@@ -1,42 +1,54 @@
+dist: xenial
+os: linux
 language: php
 
 php:
     - 5.6
     - 7.0
     - 7.1
-    - 7.2
-    - 7.3
-    - nightly
 
-matrix:
+jobs:
     include:
         - php: 7.2
-          env: CS_FIXER=run
+          env: SYMFONY_PHPUNIT_VERSION=7.5
+        - php: 7.2
+          env: SYMFONY_PHPUNIT_VERSION=7.5 COMPOSER_FLAGS="--prefer-lowest"
+        - php: 7.3
+          env: SYMFONY_PHPUNIT_VERSION=7.5 CS_FIXER=run COVERAGE=run
+        - php: 7.4
+          env: SYMFONY_PHPUNIT_VERSION=7.5
+        - php: nightly
+          env: SYMFONY_PHPUNIT_VERSION=7.5
     fast_finish: true
     allow_failures:
         - php: nightly
 
-# cache vendor dirs
 cache:
     directories:
-        - vendor
-        - $HOME/.composer/cache
+        - $HOME/.composer/cache/files
 
 before_install:
     - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi;
 
 install:
-    - composer self-update
+    # only install phpstan when we need it
+    - if [ "$CS_FIXER" = "run" ]; then composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest ; fi;
+    - composer update --prefer-dist --no-progress --no-suggest -o $COMPOSER_FLAGS
+    - php vendor/bin/simple-phpunit install
 
 before_script:
-    - if [ "$CS_FIXER" = "run" ]; then composer require phpstan/phpstan phpstan/phpstan-phpunit --dev -n ; fi;
-    - composer install -o --prefer-dist --no-interaction
+    - mkdir -p build/logs
+    - if [ "$COVERAGE" = "run" ]; then PHPUNIT_FLAGS="--coverage-clover build/logs/clover.xml"; fi;
 
 script:
-    - mkdir -p build/logs
-    - php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml
+    - php vendor/bin/simple-phpunit -v $PHPUNIT_FLAGS
     - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/php-cs-fixer fix --verbose --dry-run ; fi;
     - if [ "$CS_FIXER" = "run" ]; then php vendor/bin/phpstan analyse src tests --no-progress --level 1 ; fi;
 
-after_script:
-    - php vendor/bin/php-coveralls -v -x build/logs/clover.xml
+after_success:
+    - |
+        if [ "$COVERAGE" = "run" ]; then
+            wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.2.0/php-coveralls.phar
+            chmod +x php-coveralls.phar
+            php php-coveralls.phar -v -x build/logs/clover.xml
+        fi

composer.json

@@ -27,12 +27,11 @@
         "php": ">=5.6.0",
         "ext-mbstring": "*",
         "psr/log": "^1.0",
-        "electrolinux/php-html5lib": "^0.1.0"
+        "masterminds/html5": "^2.7"
     },
     "require-dev": {
-        "php-coveralls/php-coveralls": "^2.1",
         "friendsofphp/php-cs-fixer": "^2.14",
-        "monolog/monolog": "^1.24",
+        "monolog/monolog": "^1.24|^2.1",
         "symfony/phpunit-bridge": "^4.2.3"
     },
     "suggest": {

phpstan.neon

@@ -5,4 +5,4 @@ includes:
 parameters:
     # https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
     autoload_files:
-        - vendor/bin/.phpunit/phpunit-6.5/vendor/autoload.php
+        - vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php

phpunit.xml.dist

@@ -7,7 +7,6 @@
     convertWarningsToExceptions="true"
     processIsolation="false"
     stopOnFailure="false"
-    syntaxCheck="false"
     bootstrap="vendor/autoload.php"
     >
 
@@ -27,6 +26,6 @@
     </filter>
 
     <!-- <logging>
-        <log type="coverage-html" target="coverage" title="Readability" charset="UTF-8" yui="true" highlight="true" lowUpperBound="35" highLowerBound="70"/>
+        <log type="coverage-html" target="coverage" lowUpperBound="35" highLowerBound="70"/>
     </logging> -->
 </phpunit>

src/Readability.php

@@ -2,7 +2,7 @@
 
 namespace Readability;
 
-use HTML5Lib\Parser;
+use Masterminds\HTML5;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
@@ -61,7 +61,7 @@ class Readability implements LoggerAwareInterface
     // constants
     const SCORE_CHARS_IN_PARAGRAPH = 100;
     const SCORE_WORDS_IN_PARAGRAPH = 20;
-    const GRANDPARENT_SCORE_DIVISOR = 2.2;
+    const GRANDPARENT_SCORE_DIVISOR = 2;
     const MIN_PARAGRAPH_LENGTH = 20;
     const MIN_COMMAS_IN_PARAGRAPH = 6;
     const MIN_ARTICLE_LENGTH = 200;
@@ -92,10 +92,10 @@ class Readability implements LoggerAwareInterface
         'unlikelyCandidates' => '/display\s*:\s*none|ignore|\binfos?\b|annoy|clock|date|time|author|intro|hidd?e|about|archive|\bprint|bookmark|tags|tag-list|share|search|social|robot|published|combx|comment|mast(?:head)|subscri|community|category|disqus|extra|head|head(?:er|note)|floor|foot(?:er|note)|menu|tool\b|function|nav|remark|rss|shoutbox|widget|meta|banner|sponsor|adsense|inner-?ad|ad-|sponsor|\badv\b|\bads\b|agr?egate?|pager|sidebar|popup|tweet|twitter/i',
         'okMaybeItsACandidate' => '/article\b|contain|\bcontent|column|general|detail|shadow|lightbox|blog|body|entry|main|page|footnote/i',
         'positive' => '/read|full|article|body|\bcontent|contain|entry|main|markdown|media|page|attach|pagination|post|text|blog|story/i',
-        'negative' => '/bottom|stat|info|discuss|e[\-]?mail|comment|reply|log.{2}(n|ed)|sign|single|combx|com-|contact|_nav|link|media|\bout|promo|\bad-|related|scroll|shoutbox|sidebar|sponsor|shopping|teaser|recommend/i',
+        'negative' => '/bottom|stat|info|discuss|e[\-]?mail|comment|reply|log.{2}(n|ed)|sign|single|combx|com-|contact|_nav|link|media|promo|\bad-|related|scroll|shoutbox|sidebar|sponsor|shopping|teaser|recommend/i',
         'divToPElements' => '/<(?:blockquote|header|section|code|div|article|footer|aside|img|p|pre|dl|ol|ul)/mi',
         'killBreaks' => '/(<br\s*\/?>([ \r\n\s]|&nbsp;?)*)+/',
-        'media' => '!//(?:[^\.\?/]+\.)?(?:youtu(?:be)?|giphy|soundcloud|dailymotion|vimeo|pornhub|xvideos|twitvid|rutube|viddler)\.(?:com|be|org|net)/!i',
+        'media' => '!//(?:[^\.\?/]+\.)?(?:youtu(?:be)?|giphy|soundcloud|dailymotion|vimeo|pornhub|xvideos|twitvid|rutube|openload\.co|viddler)\.(?:com|be|org|net)/!i',
         'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i',
     ];
     public $tidy_config = [
@@ -307,8 +307,6 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Run any post-process modifications to article content as necessary.
-     *
-     * @param \DOMElement $articleContent
      */
     public function postProcessContent(\DOMElement $articleContent)
     {
@@ -321,8 +319,6 @@ class Readability implements LoggerAwareInterface
      * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
      *
      * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
-     *
-     * @param \DOMElement $articleContent
      */
     public function addFootnotes(\DOMElement $articleContent)
     {
@@ -385,8 +381,6 @@ class Readability implements LoggerAwareInterface
     /**
      * Prepare the article node for display. Clean out any inline styles,
      * iframes, forms, strip extraneous <p> tags, etc.
-     *
-     * @param \DOMNode $articleContent
      */
     public function prepArticle(\DOMNode $articleContent)
     {
@@ -558,8 +552,7 @@ class Readability implements LoggerAwareInterface
      * This is the amount of text that is inside a link divided by the total text in the node.
      * Can exclude external references to differentiate between simple text and menus/infoblocks.
      *
-     * @param \DOMElement $e
-     * @param bool        $excludeExternal
+     * @param bool $excludeExternal
      *
      * @return int
      */
@@ -586,8 +579,6 @@ class Readability implements LoggerAwareInterface
     /**
      * Get an element relative weight.
      *
-     * @param \DOMElement $e
-     *
      * @return int
      */
     public function getWeight(\DOMElement $e)
@@ -607,8 +598,6 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Remove extraneous break tags from a node.
-     *
-     * @param \DOMElement $node
      */
     public function killBreaks(\DOMElement $node)
     {
@@ -623,12 +612,10 @@ class Readability implements LoggerAwareInterface
      *
      * Updated 2012-09-18 to preserve youtube/vimeo iframes
      *
-     * @param \DOMElement $e
-     * @param string      $tag
+     * @param string $tag
      */
     public function clean(\DOMElement $e, $tag)
     {
-        $currentItem = null;
         $targetList = $e->getElementsByTagName($tag);
         $isEmbed = ('audio' === $tag || 'video' === $tag || 'iframe' === $tag || 'object' === $tag || 'embed' === $tag);
 
@@ -659,8 +646,7 @@ class Readability implements LoggerAwareInterface
      * "Fishy" is an algorithm based on content length, classnames,
      * link density, number of images & embeds, etc.
      *
-     * @param \DOMElement $e
-     * @param string      $tag
+     * @param string $tag
      */
     public function cleanConditionally(\DOMElement $e, $tag)
     {
@@ -670,7 +656,6 @@ class Readability implements LoggerAwareInterface
 
         $tagsList = $e->getElementsByTagName($tag);
         $curTagsLength = $tagsList->length;
-        $node = null;
 
         /*
          * Gather counts for other typical elements embedded within.
@@ -772,8 +757,6 @@ class Readability implements LoggerAwareInterface
 
     /**
      * Clean out spurious headers from an Element. Checks things like classnames and link density.
-     *
-     * @param \DOMElement $e
      */
     public function cleanHeaders(\DOMElement $e)
     {
@@ -915,8 +898,6 @@ class Readability implements LoggerAwareInterface
     /**
      * Initialize a node with the readability object. Also checks the
      * className/id for special names to add to its score.
-     *
-     * @param \DOMElement $node
      */
     protected function initializeNode(\DOMElement $node)
     {
@@ -1115,7 +1096,7 @@ class Readability implements LoggerAwareInterface
             // Add the score to the parent. The grandparent gets half.
             $parentNode->getAttributeNode('readability')->value += $contentScore;
             if ($grandParentNode) {
-                $grandParentNode->getAttributeNode('readability')->value += $contentScore / self::GRANDPARENT_SCORE_DIVISOR;
+                $grandParentNode->getAttributeNode('readability')->value += round($contentScore / self::GRANDPARENT_SCORE_DIVISOR);
             }
         }
 
@@ -1125,7 +1106,6 @@ class Readability implements LoggerAwareInterface
          */
         if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS) && $xpath) {
             $candidates = $xpath->query('.//*[(self::footer and count(//footer)<2) or (self::aside and count(//aside)<2)]', $page->documentElement);
-            $node = null;
 
             for ($c = $candidates->length - 1; $c >= 0; --$c) {
                 $node = $candidates->item($c);
@@ -1137,7 +1117,6 @@ class Readability implements LoggerAwareInterface
             }
 
             $candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
-            $node = null;
 
             for ($c = $candidates->length - 1; $c >= 0; --$c) {
                 $node = $candidates->item($c);
@@ -1346,8 +1325,7 @@ class Readability implements LoggerAwareInterface
      * Get an element weight by attribute.
      * Uses regular expressions to tell if this element looks good or bad.
      *
-     * @param \DOMElement $element
-     * @param string      $attribute
+     * @param string $attribute
      *
      * @return int
      */
@@ -1442,8 +1420,8 @@ class Readability implements LoggerAwareInterface
 
         $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');
 
-        if ('html5lib' === $this->parser) {
-            $this->dom = Parser::parse($this->html);
+        if ('html5lib' === $this->parser || 'html5' === $this->parser) {
+            $this->dom = (new HTML5())->loadHTML($this->html);
         }
 
         if ('libxml' === $this->parser) {

tests/ReadabilityTest.php

@@ -193,7 +193,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
         $this->assertEmpty($readability->getTitle()->getInnerHtml());
         $this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
         $this->assertNotContains('<aside>', $readability->getContent()->getInnerHtml());
-        $this->assertContains('<footer readability="4"/>', $readability->getContent()->getInnerHtml());
+        $this->assertContains('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
     }
 
     public function testWithClasses()