mirror of
https://github.com/j0k3r/php-readability.git
synced 2026-09-27 14:36:23 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5a4a490e1 | ||
|
|
908a49824f | ||
|
|
c67189248e | ||
|
|
91b80b70e2 | ||
|
|
8667dae74b | ||
|
|
eecae93161 | ||
|
|
814c6e4730 |
+4
-2
@@ -1,18 +1,20 @@
|
||||
language: php
|
||||
|
||||
php:
|
||||
- 5.3.3
|
||||
- 5.3
|
||||
- 5.4
|
||||
- 5.5
|
||||
- 5.6
|
||||
- nightly
|
||||
- hhvm-nightly
|
||||
- hhvm
|
||||
|
||||
# run build against nightly but allow them to fail
|
||||
matrix:
|
||||
fast_finish: true
|
||||
allow_failures:
|
||||
- php: nightly
|
||||
- php: hhvm-nightly
|
||||
- php: hhvm
|
||||
|
||||
# faster builds on new travis setup not using sudo
|
||||
sudo: false
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
"role": "Developer (original JS version)"
|
||||
}],
|
||||
"require": {
|
||||
"php": ">=5.4",
|
||||
"php": ">=5.3.3",
|
||||
"ext-tidy": ">=1.2"
|
||||
},
|
||||
"autoload": {
|
||||
|
||||
+1
-1
@@ -27,6 +27,6 @@
|
||||
</filter>
|
||||
|
||||
<logging>
|
||||
<log type="coverage-html" target="coverage" title="FullText" charset="UTF-8" yui="true" highlight="true" lowUpperBound="35" highLowerBound="70"/>
|
||||
<log type="coverage-html" target="coverage" title="Readability" charset="UTF-8" yui="true" highlight="true" lowUpperBound="35" highLowerBound="70"/>
|
||||
</logging>
|
||||
</phpunit>
|
||||
|
||||
@@ -47,7 +47,7 @@ class JSLikeHTMLElement extends \DOMElement
|
||||
{
|
||||
if ($name == 'innerHTML') {
|
||||
// first, empty the element
|
||||
for ($x = $this->childNodes->length - 1; $x >= 0; $x--) {
|
||||
for ($x = $this->childNodes->length - 1; $x >= 0; --$x) {
|
||||
$this->removeChild($this->childNodes->item($x));
|
||||
}
|
||||
// $value holds our new inner HTML
|
||||
|
||||
+94
-46
@@ -47,7 +47,6 @@ namespace Readability;
|
||||
*/
|
||||
class Readability
|
||||
{
|
||||
public $version = '1.7.2-without-multi-page';
|
||||
public $convertLinksToFootnotes = false;
|
||||
public $revertForcedParagraphElements = true;
|
||||
public $articleTitle;
|
||||
@@ -64,6 +63,7 @@ class Readability
|
||||
protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
|
||||
protected $flags = 7; // 1 | 2 | 4; // Start with all processing flags set.
|
||||
protected $success = false; // indicates whether we were able to extract or not
|
||||
|
||||
/**
|
||||
* All of the regular expressions in use within readability.
|
||||
* Defined up here so we don't instantiate them repeatedly in loops.
|
||||
@@ -89,9 +89,9 @@ class Readability
|
||||
'output-xhtml' => true,
|
||||
'logical-emphasis' => true,
|
||||
'show-body-only' => false,
|
||||
'new-blocklevel-tags' => 'article,aside,audio,details,figcaption,figure,footer,header,hgroup,nav,section,source,summary,temp,track,video',
|
||||
'new-empty-tags' => 'command,embed,keygen,source,track,wbr',
|
||||
'new-inline-tags' => 'audio,canvas,command,datalist,embed,keygen,mark,meter,output,progress,time,video,wbr',
|
||||
'new-blocklevel-tags' => 'article aside audio bdi canvas details dialog figcaption figure footer header hgroup main menu menuitem nav section source summary template track video',
|
||||
'new-empty-tags' => 'command embed keygen source track wbr',
|
||||
'new-inline-tags' => 'audio command datalist embed keygen mark menuitem meter output progress source time video wbr',
|
||||
'wrap' => 0,
|
||||
'drop-empty-paras' => true,
|
||||
'drop-proprietary-attributes' => false,
|
||||
@@ -123,6 +123,7 @@ class Readability
|
||||
'!</code>\s*</pre>!is' => '</pre>',
|
||||
'!<[hb]r>!is' => '<\\1 />',
|
||||
);
|
||||
|
||||
// flags
|
||||
const FLAG_STRIP_UNLIKELYS = 1;
|
||||
const FLAG_WEIGHT_ATTRIBUTES = 2;
|
||||
@@ -138,6 +139,7 @@ class Readability
|
||||
const MIN_ARTICLE_LENGTH = 200;
|
||||
const MIN_NODE_LENGTH = 80;
|
||||
const MAX_LINK_DENSITY = 0.25;
|
||||
|
||||
/**
|
||||
* Create instance of Readability.
|
||||
*
|
||||
@@ -195,12 +197,19 @@ class Readability
|
||||
libxml_use_internal_errors(true);
|
||||
$this->dom = new \DOMDocument();
|
||||
$this->dom->preserveWhiteSpace = false;
|
||||
@$this->dom->loadHTML($html, LIBXML_NOBLANKS | LIBXML_COMPACT | LIBXML_NOERROR);
|
||||
|
||||
if (PHP_VERSION_ID >= 50400) {
|
||||
$this->dom->loadHTML($html, LIBXML_NOBLANKS | LIBXML_COMPACT | LIBXML_NOERROR);
|
||||
} else {
|
||||
$this->dom->loadHTML($html);
|
||||
}
|
||||
|
||||
libxml_use_internal_errors(false);
|
||||
}
|
||||
|
||||
$this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get article title element.
|
||||
*
|
||||
@@ -210,6 +219,7 @@ class Readability
|
||||
{
|
||||
return $this->articleTitle;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get article content element.
|
||||
*
|
||||
@@ -219,6 +229,7 @@ class Readability
|
||||
{
|
||||
return $this->articleContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add pre filter for raw input HTML processing.
|
||||
*
|
||||
@@ -229,6 +240,7 @@ class Readability
|
||||
{
|
||||
$this->pre_filters[$filter] = $replacer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add post filter for raw output HTML processing.
|
||||
*
|
||||
@@ -239,6 +251,7 @@ class Readability
|
||||
{
|
||||
$this->post_filters[$filter] = $replacer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs readability.
|
||||
*
|
||||
@@ -274,7 +287,7 @@ class Readability
|
||||
// Build readability's DOM tree.
|
||||
$overlay = $this->dom->createElement('div');
|
||||
$innerDiv = $this->dom->createElement('div');
|
||||
$articleTitle = $this->getArticleTitle();
|
||||
$articleTitle = $this->getArticleTitle();
|
||||
$articleContent = $this->grabArticle();
|
||||
if (!$articleContent) {
|
||||
$this->success = false;
|
||||
@@ -300,6 +313,7 @@ class Readability
|
||||
|
||||
return $this->success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug.
|
||||
*/
|
||||
@@ -320,6 +334,7 @@ class Readability
|
||||
syslog(6, $this->debugText); // 1 - error 6 - info
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run any post-process modifications to article content as necessary.
|
||||
*
|
||||
@@ -331,6 +346,7 @@ class Readability
|
||||
$this->addFootnotes($articleContent);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the article title as an H1.
|
||||
*
|
||||
@@ -344,6 +360,7 @@ class Readability
|
||||
$curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
|
||||
} catch (Exception $e) {
|
||||
}
|
||||
|
||||
if (preg_match('/ [\|\-] /', $curTitle)) {
|
||||
$curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
|
||||
if (count(explode(' ', $curTitle)) < 3) {
|
||||
@@ -360,15 +377,18 @@ class Readability
|
||||
$curTitle = $this->getInnerText($hOnes->item(0));
|
||||
}
|
||||
}
|
||||
|
||||
$curTitle = trim($curTitle);
|
||||
if (count(explode(' ', $curTitle)) <= 4) {
|
||||
$curTitle = $origTitle;
|
||||
}
|
||||
|
||||
$articleTitle = $this->dom->createElement('h1');
|
||||
$articleTitle->innerHTML = $curTitle;
|
||||
|
||||
return $articleTitle;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the HTML document for readability to scrape it.
|
||||
* This includes things like stripping javascript, CSS, and handling terrible markup.
|
||||
@@ -386,14 +406,15 @@ class Readability
|
||||
$this->body->setAttribute('id', 'readabilityBody');
|
||||
// Remove all style tags in head.
|
||||
$styleTags = $this->dom->getElementsByTagName('style');
|
||||
for ($i = $styleTags->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $styleTags->length - 1; $i >= 0; --$i) {
|
||||
$styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
|
||||
}
|
||||
$linkTags = $this->dom->getElementsByTagName('link');
|
||||
for ($i = $linkTags->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $linkTags->length - 1; $i >= 0; --$i) {
|
||||
$linkTags->item($i)->parentNode->removeChild($linkTags->item($i));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* For easier reading, convert this document to have footnotes at the bottom rather than inline links.
|
||||
*
|
||||
@@ -409,12 +430,12 @@ class Readability
|
||||
$footnotesWrapper->appendChild($articleFootnotes);
|
||||
$articleLinks = $articleContent->getElementsByTagName('a');
|
||||
$linkCount = 0;
|
||||
for ($i = 0; $i < $articleLinks->length; $i++) {
|
||||
$articleLink = $articleLinks->item($i);
|
||||
for ($i = 0; $i < $articleLinks->length; ++$i) {
|
||||
$articleLink = $articleLinks->item($i);
|
||||
$footnoteLink = $articleLink->cloneNode(true);
|
||||
$refLink = $this->dom->createElement('a');
|
||||
$footnote = $this->dom->createElement('li');
|
||||
$linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
|
||||
$linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
|
||||
if (!$linkDomain && isset($this->url)) {
|
||||
$linkDomain = @parse_url($this->url, PHP_URL_HOST);
|
||||
}
|
||||
@@ -422,7 +443,7 @@ class Readability
|
||||
if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
|
||||
continue;
|
||||
}
|
||||
$linkCount++;
|
||||
++$linkCount;
|
||||
// Add a superscript reference after the article link.
|
||||
$refLink->setAttribute('href', '#readabilityFootnoteLink-'.$linkCount);
|
||||
$refLink->innerHTML = '<small><sup>['.$linkCount.']</sup></small>';
|
||||
@@ -448,6 +469,7 @@ class Readability
|
||||
$articleContent->appendChild($footnotesWrapper);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the article node for display. Clean out any inline styles,
|
||||
* iframes, forms, strip extraneous <p> tags, etc.
|
||||
@@ -470,20 +492,20 @@ class Readability
|
||||
* which is what they were before.
|
||||
*/
|
||||
$elems = $xpath->query('.//p[@data-readability-styled]', $articleContent);
|
||||
for ($i = $elems->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $elems->length - 1; $i >= 0; --$i) {
|
||||
$e = $elems->item($i);
|
||||
$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
|
||||
}
|
||||
}
|
||||
// Remove service data-candidate attribute.
|
||||
$elems = $xpath->query('.//*[@data-candidate]', $articleContent);
|
||||
for ($i = $elems->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $elems->length - 1; $i >= 0; --$i) {
|
||||
$elems->item($i)->removeAttribute('data-candidate');
|
||||
}
|
||||
// Remove unrelated links and other unneded stuff.
|
||||
// (not(*) and not(text()[normalize-space()])) or // What's wrong here?
|
||||
$elems = $xpath->query('.//a[@rel="nofollow"]', $articleContent);
|
||||
for ($i = $elems->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $elems->length - 1; $i >= 0; --$i) {
|
||||
$elems->item($i)->parentNode->removeChild($elems->item($i));
|
||||
}
|
||||
// Clean out junk from the article content.
|
||||
@@ -512,7 +534,7 @@ class Readability
|
||||
$this->cleanConditionally($articleContent, 'div');
|
||||
// Remove extra paragraphs.
|
||||
$articleParagraphs = $articleContent->getElementsByTagName('p');
|
||||
for ($i = $articleParagraphs->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $articleParagraphs->length - 1; $i >= 0; --$i) {
|
||||
$imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
|
||||
$embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
|
||||
$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
|
||||
@@ -542,6 +564,7 @@ class Readability
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a node with the readability object. Also checks the
|
||||
* className/id for special names to add to its score.
|
||||
@@ -609,6 +632,7 @@ class Readability
|
||||
}
|
||||
$readability->value += $this->getWeight($node);
|
||||
}
|
||||
|
||||
/**
|
||||
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
|
||||
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
|
||||
@@ -626,7 +650,7 @@ class Readability
|
||||
$xpath = new \DOMXPath($page);
|
||||
}
|
||||
$allElements = $page->getElementsByTagName('*');
|
||||
for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
|
||||
for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
|
||||
$tagName = $node->tagName;
|
||||
// Some well known site uses sections as paragraphs.
|
||||
if (strcasecmp($tagName, 'p') === 0 || strcasecmp($tagName, 'td') === 0 || strcasecmp($tagName, 'section') === 0) {
|
||||
@@ -644,14 +668,14 @@ class Readability
|
||||
//$newNode->setAttribute('class', $node->getAttribute('class'));
|
||||
//$newNode->setAttribute('id', $node->getAttribute('id'));
|
||||
$node = $node->parentNode->replaceChild($newNode, $node);
|
||||
$nodeIndex--;
|
||||
--$nodeIndex;
|
||||
$nodesToScore[] = $newNode;
|
||||
} catch (Exception $e) {
|
||||
$this->dbg('Could not alter div/article to p, reverting back to div: '.$e->getMessage());
|
||||
}
|
||||
} else {
|
||||
// Will change these P elements back to text nodes after processing.
|
||||
for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
|
||||
for ($i = 0, $il = $node->childNodes->length; $i < $il; ++$i) {
|
||||
$childNode = $node->childNodes->item($i);
|
||||
if (is_object($childNode) && get_class($childNode) === 'DOMProcessingInstruction') { //executable tags (<?php or <?xml) warning
|
||||
$childNode->parentNode->removeChild($childNode);
|
||||
@@ -675,7 +699,7 @@ class Readability
|
||||
* A score is determined by things like number of commas, class names, etc.
|
||||
* Maybe eventually link density.
|
||||
*/
|
||||
for ($pt = 0, $scored = count($nodesToScore); $pt < $scored; $pt++) {
|
||||
for ($pt = 0, $scored = count($nodesToScore); $pt < $scored; ++$pt) {
|
||||
$parentNode = $nodesToScore[$pt]->parentNode;
|
||||
// No parent node? Move on...
|
||||
if (!$parentNode) {
|
||||
@@ -730,7 +754,7 @@ class Readability
|
||||
*/
|
||||
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS) && $xpath) {
|
||||
$candidates = $xpath->query('.//*[(self::footer and count(//footer)<2) or (self::aside and count(//aside)<2)]', $page->documentElement);
|
||||
for ($node = null, $c = $candidates->length - 1; $c >= 0; $c--) {
|
||||
for ($node = null, $c = $candidates->length - 1; $c >= 0; --$c) {
|
||||
$node = $candidates->item($c);
|
||||
// node should be readable but not inside of an article otherwise it's probably non-readable block
|
||||
if ($node->hasAttribute('readability') && (int) $node->getAttributeNode('readability')->value < 40 && ($node->parentNode ? strcasecmp($node->parentNode->tagName, 'article') !== 0 : true)) {
|
||||
@@ -739,7 +763,7 @@ class Readability
|
||||
}
|
||||
}
|
||||
$candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
|
||||
for ($node = null, $c = $candidates->length - 1; $c >= 0; $c--) {
|
||||
for ($node = null, $c = $candidates->length - 1; $c >= 0; --$c) {
|
||||
$node = $candidates->item($c);
|
||||
$tagName = $node->tagName;
|
||||
/* Remove unlikely candidates */
|
||||
@@ -751,7 +775,7 @@ class Readability
|
||||
) {
|
||||
$this->dbg('Removing unlikely candidate '.$node->getNodePath().' by "'.$unlikelyMatchString.'" with readability '.($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
|
||||
$node->parentNode->removeChild($node);
|
||||
$nodeIndex--;
|
||||
--$nodeIndex;
|
||||
}
|
||||
}
|
||||
unset($candidates);
|
||||
@@ -764,7 +788,7 @@ class Readability
|
||||
if ($xpath) {
|
||||
// Using array of DOMElements after deletion is a path to DOOMElement.
|
||||
$candidates = $xpath->query('.//*[@data-candidate]', $page->documentElement);
|
||||
for ($c = $candidates->length - 1; $c >= 0; $c--) {
|
||||
for ($c = $candidates->length - 1; $c >= 0; --$c) {
|
||||
// Scale the final candidates score based on link density. Good content should have a
|
||||
// relatively small link density (5% or less) and be mostly unaffected by this operation.
|
||||
// If not for this we would have used XPath to find maximum @readability.
|
||||
@@ -791,6 +815,7 @@ class Readability
|
||||
$this->dbg('Setting body to a raw HTML of original page!');
|
||||
$topCandidate->innerHTML = $page->documentElement->innerHTML;
|
||||
$page->documentElement->innerHTML = '';
|
||||
$this->reinitBody();
|
||||
$page->documentElement->appendChild($topCandidate);
|
||||
}
|
||||
} else {
|
||||
@@ -824,7 +849,7 @@ class Readability
|
||||
$siblingNodes = new stdClass();
|
||||
$siblingNodes->length = 0;
|
||||
}
|
||||
for ($s = 0, $sl = $siblingNodes->length; $s < $sl; $s++) {
|
||||
for ($s = 0, $sl = $siblingNodes->length; $s < $sl; ++$s) {
|
||||
$siblingNode = $siblingNodes->item($s);
|
||||
$siblingNodeName = $siblingNode->nodeName;
|
||||
$append = false;
|
||||
@@ -844,7 +869,7 @@ class Readability
|
||||
if (strcasecmp($siblingNodeName, 'p') === 0) {
|
||||
$linkDensity = $this->getLinkDensity($siblingNode);
|
||||
$nodeContent = $this->getInnerText($siblingNode, true, true);
|
||||
$nodeLength = mb_strlen($nodeContent);
|
||||
$nodeLength = mb_strlen($nodeContent);
|
||||
if ($nodeLength > self::MIN_NODE_LENGTH && $linkDensity < self::MAX_LINK_DENSITY) {
|
||||
$append = true;
|
||||
} elseif ($nodeLength < self::MIN_NODE_LENGTH && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) {
|
||||
@@ -865,13 +890,13 @@ class Readability
|
||||
} catch (Exception $e) {
|
||||
$this->dbg('Could not alter siblingNode '.$siblingNodeName.' to div, reverting to original.');
|
||||
$nodeToAppend = $siblingNode;
|
||||
$s--;
|
||||
$sl--;
|
||||
--$s;
|
||||
--$sl;
|
||||
}
|
||||
} else {
|
||||
$nodeToAppend = $siblingNode;
|
||||
$s--;
|
||||
$sl--;
|
||||
--$s;
|
||||
--$sl;
|
||||
}
|
||||
// To ensure a node does not interfere with readability styles, remove its classnames & ids.
|
||||
// Now done via RegExp post_filter.
|
||||
@@ -891,10 +916,8 @@ class Readability
|
||||
* finding the -right- content.
|
||||
*/
|
||||
if (mb_strlen($this->getInnerText($articleContent, false)) < self::MIN_ARTICLE_LENGTH) {
|
||||
if (!$this->body->hasChildNodes()) {
|
||||
$this->body = $this->dom->createElement('body');
|
||||
}
|
||||
$this->body->innerHTML = $this->bodyCache;
|
||||
$this->reinitBody();
|
||||
|
||||
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
|
||||
$this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
|
||||
$this->dbg('...content is shorter than '.self::MIN_ARTICLE_LENGTH." letters, trying not to strip unlikely content.\n");
|
||||
@@ -917,6 +940,7 @@ class Readability
|
||||
|
||||
return $articleContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the inner text of a node.
|
||||
* This also strips out any excess whitespace to be found.
|
||||
@@ -941,6 +965,7 @@ class Readability
|
||||
|
||||
return $textContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the style attribute on every $e and under.
|
||||
*
|
||||
@@ -956,6 +981,7 @@ class Readability
|
||||
$elem->removeAttribute('style');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comma number for a given text.
|
||||
*
|
||||
@@ -967,6 +993,7 @@ class Readability
|
||||
{
|
||||
return substr_count($text, ',');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get words number for a given text if words separated by a space.
|
||||
* Input string should be normalized.
|
||||
@@ -979,6 +1006,7 @@ class Readability
|
||||
{
|
||||
return substr_count($text, ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the density of links as a percentage of the content
|
||||
* This is the amount of text that is inside a link divided by the total text in the node.
|
||||
@@ -994,7 +1022,7 @@ class Readability
|
||||
$links = $e->getElementsByTagName('a');
|
||||
$textLength = mb_strlen($this->getInnerText($e, true, true));
|
||||
$linkLength = 0;
|
||||
for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; $i++) {
|
||||
for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; ++$i) {
|
||||
if ($excludeExternal && $dRe && !preg_match($dRe, $links->item($i)->getAttribute('href'))) {
|
||||
continue;
|
||||
}
|
||||
@@ -1006,6 +1034,7 @@ class Readability
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an element weight by attribute.
|
||||
* Uses regular expressions to tell if this element looks good or bad.
|
||||
@@ -1040,6 +1069,7 @@ class Readability
|
||||
|
||||
return $weight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an element relative weight.
|
||||
*
|
||||
@@ -1060,6 +1090,7 @@ class Readability
|
||||
|
||||
return $weight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove extraneous break tags from a node.
|
||||
*
|
||||
@@ -1071,6 +1102,7 @@ class Readability
|
||||
$html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
|
||||
$node->innerHTML = $html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean a node of all elements of type "tag".
|
||||
* (Unless it's a youtube/vimeo video. People love movies.).
|
||||
@@ -1084,7 +1116,7 @@ class Readability
|
||||
{
|
||||
$targetList = $e->getElementsByTagName($tag);
|
||||
$isEmbed = ($tag === 'audio' || $tag === 'video' || $tag === 'iframe' || $tag === 'object' || $tag === 'embed');
|
||||
for ($cur_item = null, $y = $targetList->length - 1; $y >= 0; $y--) {
|
||||
for ($cur_item = null, $y = $targetList->length - 1; $y >= 0; --$y) {
|
||||
/* Allow youtube and vimeo videos through as people usually want to see those. */
|
||||
$cur_item = $targetList->item($y);
|
||||
if ($isEmbed) {
|
||||
@@ -1101,6 +1133,7 @@ class Readability
|
||||
$cur_item->parentNode->removeChild($cur_item);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean an element of all tags of type "tag" if they look fishy.
|
||||
* "Fishy" is an algorithm based on content length, classnames,
|
||||
@@ -1122,7 +1155,7 @@ class Readability
|
||||
*
|
||||
* TODO: Consider taking into account original contentScore here.
|
||||
*/
|
||||
for ($node = null, $i = $curTagsLength - 1; $i >= 0; $i--) {
|
||||
for ($node = null, $i = $curTagsLength - 1; $i >= 0; --$i) {
|
||||
$node = $tagsList->item($i);
|
||||
//$class = $node->getAttribute('class').' '.$node->getAttribute('id'); //debug
|
||||
$weight = $this->getWeight($node);
|
||||
@@ -1143,18 +1176,18 @@ class Readability
|
||||
$a = $node->getElementsByTagName('a')->length;
|
||||
$embedCount = 0;
|
||||
$embeds = $node->getElementsByTagName('embed');
|
||||
for ($ei = 0, $il = $embeds->length; $ei < $il; $ei++) {
|
||||
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
|
||||
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
|
||||
$embedCount++;
|
||||
++$embedCount;
|
||||
}
|
||||
}
|
||||
$embeds = $node->getElementsByTagName('iframe');
|
||||
for ($ei = 0, $il = $embeds->length; $ei < $il; $ei++) {
|
||||
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
|
||||
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
|
||||
$embedCount++;
|
||||
++$embedCount;
|
||||
}
|
||||
}
|
||||
$linkDensity = $this->getLinkDensity($node, true);
|
||||
$linkDensity = $this->getLinkDensity($node, true);
|
||||
$contentLength = mb_strlen($this->getInnerText($node));
|
||||
$toRemove = false;
|
||||
if ($this->lightClean) {
|
||||
@@ -1187,8 +1220,8 @@ class Readability
|
||||
} elseif ($input > floor($p / 3)) {
|
||||
$this->dbg(' too many <input> elements');
|
||||
$toRemove = true;
|
||||
} elseif ($contentLength < 25 && ($img === 0 || $img > 2)) {
|
||||
$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
|
||||
} elseif ($contentLength < 10 && ($img === 0 || $img > 2)) {
|
||||
$this->dbg(' content length less than 10 chars and 0 images, or more than 2 images');
|
||||
$toRemove = true;
|
||||
} elseif ($weight < 25 && $linkDensity > 0.2) {
|
||||
$this->dbg(' weight is '.$weight.' lower than 0 and link density is '.sprintf('%.2f', $linkDensity).' > 0.2');
|
||||
@@ -1209,6 +1242,7 @@ class Readability
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean out spurious headers from an Element. Checks things like classnames and link density.
|
||||
*
|
||||
@@ -1216,25 +1250,39 @@ class Readability
|
||||
*/
|
||||
public function cleanHeaders($e)
|
||||
{
|
||||
for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
|
||||
for ($headerIndex = 1; $headerIndex < 3; ++$headerIndex) {
|
||||
$headers = $e->getElementsByTagName('h'.$headerIndex);
|
||||
for ($i = $headers->length - 1; $i >= 0; $i--) {
|
||||
for ($i = $headers->length - 1; $i >= 0; --$i) {
|
||||
if ($this->getWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
|
||||
$headers->item($i)->parentNode->removeChild($headers->item($i));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function flagIsActive($flag)
|
||||
{
|
||||
return ($this->flags & $flag) > 0;
|
||||
}
|
||||
|
||||
public function addFlag($flag)
|
||||
{
|
||||
$this->flags = $this->flags | $flag;
|
||||
}
|
||||
|
||||
public function removeFlag($flag)
|
||||
{
|
||||
$this->flags = $this->flags & ~$flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Will recreate previously deleted body property.
|
||||
*/
|
||||
protected function reinitBody()
|
||||
{
|
||||
if (!isset($this->body->childNodes)) {
|
||||
$this->body = $this->dom->createElement('body');
|
||||
$this->body->innerHTML = $this->bodyCache;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,8 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertFalse($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
@@ -58,7 +60,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
@@ -69,7 +73,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
@@ -81,7 +87,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
@@ -94,7 +102,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('readabilityFootnoteLink', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('readabilityLink-3', $readability->getContent()->innerHTML);
|
||||
@@ -109,7 +119,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('will be removed', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('<h2>', $readability->getContent()->innerHTML);
|
||||
@@ -123,7 +135,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('nofollow', $readability->getContent()->innerHTML);
|
||||
}
|
||||
@@ -136,7 +150,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('nofollow', $readability->getContent()->innerHTML);
|
||||
}
|
||||
@@ -149,7 +165,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('<aside>', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('<footer/>', $readability->getContent()->innerHTML);
|
||||
@@ -163,7 +181,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text should be removed', $readability->getContent()->innerHTML);
|
||||
}
|
||||
@@ -176,7 +196,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="tr"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
@@ -188,7 +210,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
@@ -201,7 +225,69 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEmpty($readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
public function testTitle()
|
||||
{
|
||||
$readability = new ReadabilityTested('<title>this is my title</title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
|
||||
$readability->debug = true;
|
||||
$res = $readability->init();
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEquals('this is my title', $readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
public function testTitleWithDash()
|
||||
{
|
||||
$readability = new ReadabilityTested('<title> title2 - title3 </title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
|
||||
$readability->debug = true;
|
||||
$res = $readability->init();
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEquals('title2 - title3', $readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
public function testTitleWithDoubleDot()
|
||||
{
|
||||
$readability = new ReadabilityTested('<title> title2 : title3 </title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
|
||||
$readability->debug = true;
|
||||
$res = $readability->init();
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEquals('title2 : title3', $readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
public function testTitleTooShortUseH1()
|
||||
{
|
||||
$readability = new ReadabilityTested('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
|
||||
$readability->debug = true;
|
||||
$res = $readability->init();
|
||||
|
||||
$this->assertTrue($res);
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
|
||||
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
|
||||
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
|
||||
$this->assertEquals('this is my h1 title !', $readability->getTitle()->innerHTML);
|
||||
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
|
||||
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user