Compare commits

...
5 Commits
3 changed files with 187 additions and 53 deletions
+1 -1
View File
@@ -47,7 +47,7 @@ class JSLikeHTMLElement extends \DOMElement
{
if ($name == 'innerHTML') {
// first, empty the element
for ($x = $this->childNodes->length - 1; $x >= 0; $x--) {
for ($x = $this->childNodes->length - 1; $x >= 0; --$x) {
$this->removeChild($this->childNodes->item($x));
}
// $value holds our new inner HTML
+94 -46
View File
@@ -63,6 +63,7 @@ class Readability
protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
protected $flags = 7; // 1 | 2 | 4; // Start with all processing flags set.
protected $success = false; // indicates whether we were able to extract or not
/**
* All of the regular expressions in use within readability.
* Defined up here so we don't instantiate them repeatedly in loops.
@@ -88,9 +89,9 @@ class Readability
'output-xhtml' => true,
'logical-emphasis' => true,
'show-body-only' => false,
'new-blocklevel-tags' => 'article,aside,audio,details,figcaption,figure,footer,header,hgroup,nav,section,source,summary,temp,track,video',
'new-empty-tags' => 'command,embed,keygen,source,track,wbr',
'new-inline-tags' => 'audio,canvas,command,datalist,embed,keygen,mark,meter,output,progress,time,video,wbr',
'new-blocklevel-tags' => 'article aside audio bdi canvas details dialog figcaption figure footer header hgroup main menu menuitem nav section source summary template track video',
'new-empty-tags' => 'command embed keygen source track wbr',
'new-inline-tags' => 'audio command datalist embed keygen mark menuitem meter output progress source time video wbr',
'wrap' => 0,
'drop-empty-paras' => true,
'drop-proprietary-attributes' => false,
@@ -122,6 +123,7 @@ class Readability
'!</code>\s*</pre>!is' => '</pre>',
'!<[hb]r>!is' => '<\\1 />',
);
// flags
const FLAG_STRIP_UNLIKELYS = 1;
const FLAG_WEIGHT_ATTRIBUTES = 2;
@@ -137,6 +139,7 @@ class Readability
const MIN_ARTICLE_LENGTH = 200;
const MIN_NODE_LENGTH = 80;
const MAX_LINK_DENSITY = 0.25;
/**
* Create instance of Readability.
*
@@ -192,8 +195,10 @@ class Readability
if (!($parser == 'html5lib' && ($this->dom = \HTML5_Parser::parse($html)))) {
libxml_use_internal_errors(true);
$this->dom = new \DOMDocument();
$this->dom->preserveWhiteSpace = false;
$this->dom->formatOutput = true;
if (PHP_VERSION_ID >= 50400) {
$this->dom->loadHTML($html, LIBXML_NOBLANKS | LIBXML_COMPACT | LIBXML_NOERROR);
@@ -206,6 +211,7 @@ class Readability
$this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
}
/**
* Get article title element.
*
@@ -215,6 +221,7 @@ class Readability
{
return $this->articleTitle;
}
/**
* Get article content element.
*
@@ -224,6 +231,7 @@ class Readability
{
return $this->articleContent;
}
/**
* Add pre filter for raw input HTML processing.
*
@@ -234,6 +242,7 @@ class Readability
{
$this->pre_filters[$filter] = $replacer;
}
/**
* Add post filter for raw output HTML processing.
*
@@ -244,6 +253,7 @@ class Readability
{
$this->post_filters[$filter] = $replacer;
}
/**
* Runs readability.
*
@@ -284,11 +294,11 @@ class Readability
if (!$articleContent) {
$this->success = false;
$articleContent = $this->dom->createElement('div');
$articleContent->setAttribute('id', 'readability-content');
$articleContent->setAttribute('class', 'readability-content');
$articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
}
$overlay->setAttribute('id', 'readOverlay');
$innerDiv->setAttribute('id', 'readInner');
$overlay->setAttribute('class', 'readOverlay');
$innerDiv->setAttribute('class', 'readInner');
// Glue the structure of our document together.
$innerDiv->appendChild($articleTitle);
$innerDiv->appendChild($articleContent);
@@ -305,6 +315,7 @@ class Readability
return $this->success;
}
/**
* Debug.
*/
@@ -325,6 +336,7 @@ class Readability
syslog(6, $this->debugText); // 1 - error 6 - info
}
}
/**
* Run any post-process modifications to article content as necessary.
*
@@ -336,6 +348,7 @@ class Readability
$this->addFootnotes($articleContent);
}
}
/**
* Get the article title as an H1.
*
@@ -349,6 +362,7 @@ class Readability
$curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
} catch (Exception $e) {
}
if (preg_match('/ [\|\-] /', $curTitle)) {
$curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
if (count(explode(' ', $curTitle)) < 3) {
@@ -365,15 +379,18 @@ class Readability
$curTitle = $this->getInnerText($hOnes->item(0));
}
}
$curTitle = trim($curTitle);
if (count(explode(' ', $curTitle)) <= 4) {
$curTitle = $origTitle;
}
$articleTitle = $this->dom->createElement('h1');
$articleTitle->innerHTML = $curTitle;
return $articleTitle;
}
/**
* Prepare the HTML document for readability to scrape it.
* This includes things like stripping javascript, CSS, and handling terrible markup.
@@ -388,17 +405,18 @@ class Readability
$this->body = $this->dom->createElement('body');
$this->dom->documentElement->appendChild($this->body);
}
$this->body->setAttribute('id', 'readabilityBody');
$this->body->setAttribute('class', 'readabilityBody');
// Remove all style tags in head.
$styleTags = $this->dom->getElementsByTagName('style');
for ($i = $styleTags->length - 1; $i >= 0; $i--) {
for ($i = $styleTags->length - 1; $i >= 0; --$i) {
$styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
}
$linkTags = $this->dom->getElementsByTagName('link');
for ($i = $linkTags->length - 1; $i >= 0; $i--) {
for ($i = $linkTags->length - 1; $i >= 0; --$i) {
$linkTags->item($i)->parentNode->removeChild($linkTags->item($i));
}
}
/**
* For easier reading, convert this document to have footnotes at the bottom rather than inline links.
*
@@ -407,14 +425,14 @@ class Readability
public function addFootnotes($articleContent)
{
$footnotesWrapper = $this->dom->createElement('footer');
$footnotesWrapper->setAttribute('id', 'readability-footnotes');
$footnotesWrapper->setAttribute('class', 'readability-footnotes');
$footnotesWrapper->innerHTML = '<h3>References</h3>';
$articleFootnotes = $this->dom->createElement('ol');
$articleFootnotes->setAttribute('id', 'readability-footnotes-list');
$articleFootnotes->setAttribute('class', 'readability-footnotes-list');
$footnotesWrapper->appendChild($articleFootnotes);
$articleLinks = $articleContent->getElementsByTagName('a');
$linkCount = 0;
for ($i = 0; $i < $articleLinks->length; $i++) {
for ($i = 0; $i < $articleLinks->length; ++$i) {
$articleLink = $articleLinks->item($i);
$footnoteLink = $articleLink->cloneNode(true);
$refLink = $this->dom->createElement('a');
@@ -427,7 +445,7 @@ class Readability
if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
continue;
}
$linkCount++;
++$linkCount;
// Add a superscript reference after the article link.
$refLink->setAttribute('href', '#readabilityFootnoteLink-'.$linkCount);
$refLink->innerHTML = '<small><sup>['.$linkCount.']</sup></small>';
@@ -453,6 +471,7 @@ class Readability
$articleContent->appendChild($footnotesWrapper);
}
}
/**
* Prepare the article node for display. Clean out any inline styles,
* iframes, forms, strip extraneous <p> tags, etc.
@@ -475,20 +494,20 @@ class Readability
* which is what they were before.
*/
$elems = $xpath->query('.//p[@data-readability-styled]', $articleContent);
for ($i = $elems->length - 1; $i >= 0; $i--) {
for ($i = $elems->length - 1; $i >= 0; --$i) {
$e = $elems->item($i);
$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
}
}
// Remove service data-candidate attribute.
$elems = $xpath->query('.//*[@data-candidate]', $articleContent);
for ($i = $elems->length - 1; $i >= 0; $i--) {
for ($i = $elems->length - 1; $i >= 0; --$i) {
$elems->item($i)->removeAttribute('data-candidate');
}
// Remove unrelated links and other unneded stuff.
// (not(*) and not(text()[normalize-space()])) or // What's wrong here?
$elems = $xpath->query('.//a[@rel="nofollow"]', $articleContent);
for ($i = $elems->length - 1; $i >= 0; $i--) {
for ($i = $elems->length - 1; $i >= 0; --$i) {
$elems->item($i)->parentNode->removeChild($elems->item($i));
}
// Clean out junk from the article content.
@@ -517,7 +536,7 @@ class Readability
$this->cleanConditionally($articleContent, 'div');
// Remove extra paragraphs.
$articleParagraphs = $articleContent->getElementsByTagName('p');
for ($i = $articleParagraphs->length - 1; $i >= 0; $i--) {
for ($i = $articleParagraphs->length - 1; $i >= 0; --$i) {
$imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
$embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
@@ -547,6 +566,7 @@ class Readability
}
}
}
/**
* Initialize a node with the readability object. Also checks the
* className/id for special names to add to its score.
@@ -614,6 +634,7 @@ class Readability
}
$readability->value += $this->getWeight($node);
}
/**
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
@@ -631,7 +652,7 @@ class Readability
$xpath = new \DOMXPath($page);
}
$allElements = $page->getElementsByTagName('*');
for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
$tagName = $node->tagName;
// Some well known site uses sections as paragraphs.
if (strcasecmp($tagName, 'p') === 0 || strcasecmp($tagName, 'td') === 0 || strcasecmp($tagName, 'section') === 0) {
@@ -649,14 +670,14 @@ class Readability
//$newNode->setAttribute('class', $node->getAttribute('class'));
//$newNode->setAttribute('id', $node->getAttribute('id'));
$node = $node->parentNode->replaceChild($newNode, $node);
$nodeIndex--;
--$nodeIndex;
$nodesToScore[] = $newNode;
} catch (Exception $e) {
$this->dbg('Could not alter div/article to p, reverting back to div: '.$e->getMessage());
}
} else {
// Will change these P elements back to text nodes after processing.
for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
for ($i = 0, $il = $node->childNodes->length; $i < $il; ++$i) {
$childNode = $node->childNodes->item($i);
if (is_object($childNode) && get_class($childNode) === 'DOMProcessingInstruction') { //executable tags (<?php or <?xml) warning
$childNode->parentNode->removeChild($childNode);
@@ -680,7 +701,7 @@ class Readability
* A score is determined by things like number of commas, class names, etc.
* Maybe eventually link density.
*/
for ($pt = 0, $scored = count($nodesToScore); $pt < $scored; $pt++) {
for ($pt = 0, $scored = count($nodesToScore); $pt < $scored; ++$pt) {
$parentNode = $nodesToScore[$pt]->parentNode;
// No parent node? Move on...
if (!$parentNode) {
@@ -735,7 +756,7 @@ class Readability
*/
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS) && $xpath) {
$candidates = $xpath->query('.//*[(self::footer and count(//footer)<2) or (self::aside and count(//aside)<2)]', $page->documentElement);
for ($node = null, $c = $candidates->length - 1; $c >= 0; $c--) {
for ($node = null, $c = $candidates->length - 1; $c >= 0; --$c) {
$node = $candidates->item($c);
// node should be readable but not inside of an article otherwise it's probably non-readable block
if ($node->hasAttribute('readability') && (int) $node->getAttributeNode('readability')->value < 40 && ($node->parentNode ? strcasecmp($node->parentNode->tagName, 'article') !== 0 : true)) {
@@ -744,7 +765,7 @@ class Readability
}
}
$candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
for ($node = null, $c = $candidates->length - 1; $c >= 0; $c--) {
for ($node = null, $c = $candidates->length - 1; $c >= 0; --$c) {
$node = $candidates->item($c);
$tagName = $node->tagName;
/* Remove unlikely candidates */
@@ -756,7 +777,7 @@ class Readability
) {
$this->dbg('Removing unlikely candidate '.$node->getNodePath().' by "'.$unlikelyMatchString.'" with readability '.($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
$node->parentNode->removeChild($node);
$nodeIndex--;
--$nodeIndex;
}
}
unset($candidates);
@@ -769,7 +790,7 @@ class Readability
if ($xpath) {
// Using array of DOMElements after deletion is a path to DOOMElement.
$candidates = $xpath->query('.//*[@data-candidate]', $page->documentElement);
for ($c = $candidates->length - 1; $c >= 0; $c--) {
for ($c = $candidates->length - 1; $c >= 0; --$c) {
// Scale the final candidates score based on link density. Good content should have a
// relatively small link density (5% or less) and be mostly unaffected by this operation.
// If not for this we would have used XPath to find maximum @readability.
@@ -796,6 +817,7 @@ class Readability
$this->dbg('Setting body to a raw HTML of original page!');
$topCandidate->innerHTML = $page->documentElement->innerHTML;
$page->documentElement->innerHTML = '';
$this->reinitBody();
$page->documentElement->appendChild($topCandidate);
}
} else {
@@ -822,14 +844,14 @@ class Readability
* Things like preambles, content split by ads that we removed, etc.
*/
$articleContent = $this->dom->createElement('div');
$articleContent->setAttribute('id', 'readability-content');
$articleContent->setAttribute('class', 'readability-content');
$siblingScoreThreshold = max(10, ((int) $topCandidate->getAttribute('readability')) * 0.2);
$siblingNodes = $topCandidate->parentNode->childNodes;
if (!isset($siblingNodes)) {
$siblingNodes = new stdClass();
$siblingNodes->length = 0;
}
for ($s = 0, $sl = $siblingNodes->length; $s < $sl; $s++) {
for ($s = 0, $sl = $siblingNodes->length; $s < $sl; ++$s) {
$siblingNode = $siblingNodes->item($s);
$siblingNodeName = $siblingNode->nodeName;
$append = false;
@@ -864,19 +886,22 @@ class Readability
$this->dbg('Altering siblingNode '.$siblingNodeName.' to div.');
$nodeToAppend = $this->dom->createElement('div');
try {
if ($siblingNode->getAttribute('id')) {
$nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
}
$nodeToAppend->setAttribute('alt', $siblingNodeName);
$nodeToAppend->innerHTML = $siblingNode->innerHTML;
} catch (Exception $e) {
$this->dbg('Could not alter siblingNode '.$siblingNodeName.' to div, reverting to original.');
$nodeToAppend = $siblingNode;
$s--;
$sl--;
--$s;
--$sl;
}
} else {
$nodeToAppend = $siblingNode;
$s--;
$sl--;
--$s;
--$sl;
}
// To ensure a node does not interfere with readability styles, remove its classnames & ids.
// Now done via RegExp post_filter.
@@ -896,10 +921,8 @@ class Readability
* finding the -right- content.
*/
if (mb_strlen($this->getInnerText($articleContent, false)) < self::MIN_ARTICLE_LENGTH) {
if (!$this->body->hasChildNodes()) {
$this->body = $this->dom->createElement('body');
}
$this->body->innerHTML = $this->bodyCache;
$this->reinitBody();
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
$this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
$this->dbg('...content is shorter than '.self::MIN_ARTICLE_LENGTH." letters, trying not to strip unlikely content.\n");
@@ -922,6 +945,7 @@ class Readability
return $articleContent;
}
/**
* Get the inner text of a node.
* This also strips out any excess whitespace to be found.
@@ -946,6 +970,7 @@ class Readability
return $textContent;
}
/**
* Remove the style attribute on every $e and under.
*
@@ -961,6 +986,7 @@ class Readability
$elem->removeAttribute('style');
}
}
/**
* Get comma number for a given text.
*
@@ -972,6 +998,7 @@ class Readability
{
return substr_count($text, ',');
}
/**
* Get words number for a given text if words separated by a space.
* Input string should be normalized.
@@ -984,6 +1011,7 @@ class Readability
{
return substr_count($text, ' ');
}
/**
* Get the density of links as a percentage of the content
* This is the amount of text that is inside a link divided by the total text in the node.
@@ -999,7 +1027,7 @@ class Readability
$links = $e->getElementsByTagName('a');
$textLength = mb_strlen($this->getInnerText($e, true, true));
$linkLength = 0;
for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; $i++) {
for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; ++$i) {
if ($excludeExternal && $dRe && !preg_match($dRe, $links->item($i)->getAttribute('href'))) {
continue;
}
@@ -1011,6 +1039,7 @@ class Readability
return 0;
}
}
/**
* Get an element weight by attribute.
* Uses regular expressions to tell if this element looks good or bad.
@@ -1045,6 +1074,7 @@ class Readability
return $weight;
}
/**
* Get an element relative weight.
*
@@ -1065,6 +1095,7 @@ class Readability
return $weight;
}
/**
* Remove extraneous break tags from a node.
*
@@ -1076,6 +1107,7 @@ class Readability
$html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
$node->innerHTML = $html;
}
/**
* Clean a node of all elements of type "tag".
* (Unless it's a youtube/vimeo video. People love movies.).
@@ -1089,7 +1121,7 @@ class Readability
{
$targetList = $e->getElementsByTagName($tag);
$isEmbed = ($tag === 'audio' || $tag === 'video' || $tag === 'iframe' || $tag === 'object' || $tag === 'embed');
for ($cur_item = null, $y = $targetList->length - 1; $y >= 0; $y--) {
for ($cur_item = null, $y = $targetList->length - 1; $y >= 0; --$y) {
/* Allow youtube and vimeo videos through as people usually want to see those. */
$cur_item = $targetList->item($y);
if ($isEmbed) {
@@ -1106,6 +1138,7 @@ class Readability
$cur_item->parentNode->removeChild($cur_item);
}
}
/**
* Clean an element of all tags of type "tag" if they look fishy.
* "Fishy" is an algorithm based on content length, classnames,
@@ -1127,7 +1160,7 @@ class Readability
*
* TODO: Consider taking into account original contentScore here.
*/
for ($node = null, $i = $curTagsLength - 1; $i >= 0; $i--) {
for ($node = null, $i = $curTagsLength - 1; $i >= 0; --$i) {
$node = $tagsList->item($i);
//$class = $node->getAttribute('class').' '.$node->getAttribute('id'); //debug
$weight = $this->getWeight($node);
@@ -1148,15 +1181,15 @@ class Readability
$a = $node->getElementsByTagName('a')->length;
$embedCount = 0;
$embeds = $node->getElementsByTagName('embed');
for ($ei = 0, $il = $embeds->length; $ei < $il; $ei++) {
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
$embedCount++;
++$embedCount;
}
}
$embeds = $node->getElementsByTagName('iframe');
for ($ei = 0, $il = $embeds->length; $ei < $il; $ei++) {
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
$embedCount++;
++$embedCount;
}
}
$linkDensity = $this->getLinkDensity($node, true);
@@ -1192,8 +1225,8 @@ class Readability
} elseif ($input > floor($p / 3)) {
$this->dbg(' too many <input> elements');
$toRemove = true;
} elseif ($contentLength < 25 && ($img === 0 || $img > 2)) {
$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
} elseif ($contentLength < 10 && ($img === 0 || $img > 2)) {
$this->dbg(' content length less than 10 chars and 0 images, or more than 2 images');
$toRemove = true;
} elseif ($weight < 25 && $linkDensity > 0.2) {
$this->dbg(' weight is '.$weight.' lower than 0 and link density is '.sprintf('%.2f', $linkDensity).' > 0.2');
@@ -1214,6 +1247,7 @@ class Readability
}
}
}
/**
* Clean out spurious headers from an Element. Checks things like classnames and link density.
*
@@ -1221,25 +1255,39 @@ class Readability
*/
public function cleanHeaders($e)
{
for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
for ($headerIndex = 1; $headerIndex < 3; ++$headerIndex) {
$headers = $e->getElementsByTagName('h'.$headerIndex);
for ($i = $headers->length - 1; $i >= 0; $i--) {
for ($i = $headers->length - 1; $i >= 0; --$i) {
if ($this->getWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
$headers->item($i)->parentNode->removeChild($headers->item($i));
}
}
}
}
public function flagIsActive($flag)
{
return ($this->flags & $flag) > 0;
}
public function addFlag($flag)
{
$this->flags = $this->flags | $flag;
}
public function removeFlag($flag)
{
$this->flags = $this->flags & ~$flag;
}
/**
* Will recreate previously deleted body property.
*/
protected function reinitBody()
{
if (!isset($this->body->childNodes)) {
$this->body = $this->dom->createElement('body');
$this->body->innerHTML = $this->bodyCache;
}
}
}
+86
View File
@@ -48,6 +48,8 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertFalse($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->innerHTML);
}
@@ -58,7 +60,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
}
@@ -69,7 +73,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
}
@@ -81,7 +87,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is the awesome content :)', $readability->getContent()->innerHTML);
}
@@ -94,7 +102,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertContains('readabilityFootnoteLink', $readability->getContent()->innerHTML);
$this->assertContains('readabilityLink-3', $readability->getContent()->innerHTML);
@@ -109,7 +119,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('will be removed', $readability->getContent()->innerHTML);
$this->assertNotContains('<h2>', $readability->getContent()->innerHTML);
@@ -123,7 +135,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('<div readability=', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertContains('nofollow', $readability->getContent()->innerHTML);
}
@@ -136,7 +150,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertContains('nofollow', $readability->getContent()->innerHTML);
}
@@ -149,7 +165,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('<aside>', $readability->getContent()->innerHTML);
$this->assertContains('<footer/>', $readability->getContent()->innerHTML);
@@ -163,7 +181,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text should be removed', $readability->getContent()->innerHTML);
}
@@ -176,7 +196,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="tr"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
}
@@ -188,7 +210,9 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}
@@ -201,7 +225,69 @@ class ReadabilityTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEmpty($readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}
public function testTitle()
{
$readability = new ReadabilityTested('<title>this is my title</title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEquals('this is my title', $readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}
public function testTitleWithDash()
{
$readability = new ReadabilityTested('<title> title2 - title3 </title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEquals('title2 - title3', $readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}
public function testTitleWithDoubleDot()
{
$readability = new ReadabilityTested('<title> title2 : title3 </title><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEquals('title2 : title3', $readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}
public function testTitleTooShortUseH1()
{
$readability = new ReadabilityTested('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">'.str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7).'<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertContains('alt="article"', $readability->getContent()->innerHTML);
$this->assertEquals('this is my h1 title !', $readability->getTitle()->innerHTML);
$this->assertContains('This is an awesome text with some links, here there are', $readability->getContent()->innerHTML);
$this->assertNotContains('This text is also an awesome text and you should know that', $readability->getContent()->innerHTML);
}