mirror of
https://github.com/j0k3r/php-readability.git
synced 2026-09-26 22:16:12 +00:00
Kevin Decherf
eb72a315c4
See 'Tag omission' https://developer.mozilla.org/en-US/docs/Web/HTML/Element/figure Signed-off-by: Kevin Decherf <kevin@kdecherf.com>
1450 lines
58 KiB
PHP
1450 lines
58 KiB
PHP
<?php
|
|
|
|
namespace Readability;
|
|
|
|
use Masterminds\HTML5;
|
|
use Psr\Log\LoggerAwareInterface;
|
|
use Psr\Log\LoggerInterface;
|
|
use Psr\Log\NullLogger;
|
|
|
|
/**
|
|
* Arc90's Readability ported to PHP for FiveFilters.org
|
|
* Based on readability.js version 1.7.1 (without multi-page support)
|
|
* ------------------------------------------------------
|
|
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
|
|
* Arc90's project URL: http://lab.arc90.com/experiments/readability/
|
|
* JS Source: http://code.google.com/p/arc90labs-readability
|
|
* Ported by: Keyvan Minoukadeh, http://www.keyvan.net
|
|
* Modded by: Dither, https://dithersky.wordpress.com
|
|
* More information: http://fivefilters.org/content-only/
|
|
* License: Apache License, Version 2.0
|
|
* Requires: PHP version 5.2.0+
|
|
* Date: 2013-08-02.
|
|
*
|
|
* Differences between the PHP port and the original
|
|
* ------------------------------------------------------
|
|
* Arc90's Readability is designed to run in the browser. It works on the DOM
|
|
* tree (the parsed HTML) after the page's CSS styles have been applied and
|
|
* Javascript code executed. This PHP port does not run inside a browser.
|
|
* We use PHP's ability to parse HTML to build our DOM tree, but we cannot
|
|
* rely on CSS or Javascript support. As such, the results will not always
|
|
* match Arc90's Readability. (For example, if a web page contains CSS style
|
|
* rules or Javascript code which hide certain HTML elements from display,
|
|
* Arc90's Readability will dismiss those from consideration but our PHP port,
|
|
* unable to understand CSS or Javascript, will not know any better.)
|
|
*
|
|
* Another significant difference is that the aim of Arc90's Readability is
|
|
* to re-present the main content block of a given web page so users can
|
|
* read it more easily in their browsers. Correct identification, clean up,
|
|
* and separation of the content block is only a part of this process.
|
|
* This PHP port is only concerned with this part, it does not include code
|
|
* that relates to presentation in the browser - Arc90 already do
|
|
* that extremely well, and for PDF output there's FiveFilters.org's
|
|
* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
|
|
*
|
|
* Finally, this class contains methods that might be useful for developers
|
|
* working on HTML document fragments. So without deviating too much from
|
|
* the original code (which I don't want to do because it makes debugging
|
|
* and updating more difficult), I've tried to make it a little more
|
|
* developer friendly. You should be able to use the methods here on
|
|
* existing DOMElement objects without passing an entire HTML document to
|
|
* be parsed.
|
|
*/
|
|
class Readability implements LoggerAwareInterface
|
|
{
|
|
// flags
|
|
const FLAG_STRIP_UNLIKELYS = 1;
|
|
const FLAG_WEIGHT_ATTRIBUTES = 2;
|
|
const FLAG_CLEAN_CONDITIONALLY = 4;
|
|
const FLAG_DISABLE_PREFILTER = 8;
|
|
const FLAG_DISABLE_POSTFILTER = 16;
|
|
// constants
|
|
const SCORE_CHARS_IN_PARAGRAPH = 100;
|
|
const SCORE_WORDS_IN_PARAGRAPH = 20;
|
|
const GRANDPARENT_SCORE_DIVISOR = 2;
|
|
const MIN_PARAGRAPH_LENGTH = 20;
|
|
const MIN_COMMAS_IN_PARAGRAPH = 6;
|
|
const MIN_ARTICLE_LENGTH = 200;
|
|
const MIN_NODE_LENGTH = 80;
|
|
const MAX_LINK_DENSITY = 0.25;
|
|
public $convertLinksToFootnotes = false;
|
|
public $revertForcedParagraphElements = true;
|
|
public $articleTitle;
|
|
public $articleContent;
|
|
public $original_html;
|
|
/**
|
|
* @var \DOMDocument
|
|
*/
|
|
public $dom;
|
|
// optional - URL where HTML was retrieved
|
|
public $url = null;
|
|
// preserves more content (experimental)
|
|
public $lightClean = true;
|
|
// no more used, keept to avoid BC
|
|
public $debug = false;
|
|
public $tidied = false;
|
|
|
|
/**
|
|
* All of the regular expressions in use within readability.
|
|
* Defined up here so we don't instantiate them repeatedly in loops.
|
|
*/
|
|
public $regexps = [
|
|
'unlikelyCandidates' => '/display\s*:\s*none|ignore|\binfos?\b|annoy|clock|date|time|author|intro|hidd?e|about|archive|\bprint|bookmark|tags|tag-list|share|search|social|robot|published|combx|comment|mast(?:head)|subscri|community|category|disqus|extra|head|head(?:er|note)|floor|foot(?:er|note)|menu|tool\b|function|nav|remark|rss|shoutbox|widget|meta|banner|sponsor|adsense|inner-?ad|ad-|sponsor|\badv\b|\bads\b|agr?egate?|pager|sidebar|popup|tweet|twitter/i',
|
|
'okMaybeItsACandidate' => '/article\b|contain|\bcontent|column|general|detail|shadow|lightbox|blog|body|entry|main|page|footnote/i',
|
|
'positive' => '/read|full|article|body|\bcontent|contain|entry|main|markdown|media|page|attach|pagination|post|text|blog|story/i',
|
|
'negative' => '/bottom|stat|info|discuss|e[\-]?mail|comment|reply|log.{2}(n|ed)|sign|single|combx|com-|contact|_nav|link|media|promo|\bad-|related|scroll|shoutbox|sidebar|sponsor|shopping|teaser|recommend/i',
|
|
'divToPElements' => '/<(?:blockquote|header|section|code|div|article|footer|aside|img|p|pre|dl|ol|ul)/mi',
|
|
'killBreaks' => '/(<br\s*\/?>([ \r\n\s]| ?)*)+/',
|
|
'media' => '!//(?:[^\.\?/]+\.)?(?:youtu(?:be)?|giphy|soundcloud|dailymotion|vimeo|pornhub|xvideos|twitvid|rutube|openload\.co|viddler)\.(?:com|be|org|net)/!i',
|
|
'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i',
|
|
];
|
|
public $tidy_config = [
|
|
'tidy-mark' => false,
|
|
'vertical-space' => false,
|
|
'doctype' => 'omit',
|
|
'numeric-entities' => false,
|
|
// 'preserve-entities' => true,
|
|
'break-before-br' => false,
|
|
'clean' => true,
|
|
'output-xhtml' => true,
|
|
'logical-emphasis' => true,
|
|
'show-body-only' => false,
|
|
'new-blocklevel-tags' => 'article aside audio bdi canvas details dialog figcaption figure footer header hgroup main menu menuitem nav section source summary template track video',
|
|
'new-empty-tags' => 'command embed keygen source track wbr',
|
|
'new-inline-tags' => 'audio command datalist embed keygen mark menuitem meter output progress source time video wbr',
|
|
'wrap' => 0,
|
|
'drop-empty-paras' => true,
|
|
'drop-proprietary-attributes' => false,
|
|
'enclose-text' => true,
|
|
'merge-divs' => true,
|
|
// 'merge-spans' => true,
|
|
'input-encoding' => '????',
|
|
'output-encoding' => 'utf8',
|
|
'hide-comments' => true,
|
|
];
|
|
// article domain regexp for calibration
|
|
protected $domainRegExp = null;
|
|
protected $body = null;
|
|
// Cache the body HTML in case we need to re-use it later
|
|
protected $bodyCache = null;
|
|
// 1 | 2 | 4; // Start with all processing flags set.
|
|
protected $flags = 7;
|
|
// indicates whether we were able to extract or not
|
|
protected $success = false;
|
|
protected $logger;
|
|
protected $parser;
|
|
protected $html;
|
|
protected $useTidy;
|
|
// raw HTML filters
|
|
protected $pre_filters = [
|
|
// remove obvious scripts
|
|
'!<script[^>]*>(.*?)</script>!is' => '',
|
|
// remove obvious styles
|
|
'!<style[^>]*>(.*?)</style>!is' => '',
|
|
// remove spans as we redefine styles and they're probably special-styled
|
|
'!</?span[^>]*>!is' => '',
|
|
// HACK: firewall-filtered content
|
|
'!<font[^>]*>\s*\[AD\]\s*</font>!is' => '',
|
|
// HACK: replace linebreaks plus br's with p's
|
|
'!(<br[^>]*>[ \r\n\s]*){2,}!i' => '</p><p>',
|
|
// replace noscripts
|
|
//'!</?noscript>!is' => '',
|
|
// replace fonts to spans
|
|
'!<(/?)font[^>]*>!is' => '<\\1span>',
|
|
];
|
|
// output HTML filters
|
|
protected $post_filters = [
|
|
// replace excessive br's
|
|
'/<br\s*\/?>\s*<p/i' => '<p',
|
|
// replace empty tags that break layouts
|
|
'!<(?:a|div|p|figure)[^>]+/>!is' => '',
|
|
// remove all attributes on text tags
|
|
//'!<(\s*/?\s*(?:blockquote|br|hr|code|div|article|span|footer|aside|p|pre|dl|li|ul|ol)) [^>]+>!is' => "<\\1>",
|
|
//single newlines cleanup
|
|
"/\n+/" => "\n",
|
|
// modern web...
|
|
'!<pre[^>]*>\s*<code!is' => '<pre',
|
|
'!</code>\s*</pre>!is' => '</pre>',
|
|
'!<[hb]r>!is' => '<\\1 />',
|
|
];
|
|
|
|
/**
|
|
* Create instance of Readability.
|
|
*
|
|
* @param string $html UTF-8 encoded string
|
|
* @param string $url URL associated with HTML (for footnotes)
|
|
* @param string $parser Which parser to use for turning raw HTML into a DOMDocument
|
|
* @param bool $use_tidy Use tidy
|
|
*/
|
|
public function __construct($html, $url = null, $parser = 'libxml', $use_tidy = true)
|
|
{
|
|
$this->url = $url;
|
|
$this->html = $html;
|
|
$this->parser = $parser;
|
|
$this->useTidy = $use_tidy && \function_exists('tidy_parse_string');
|
|
|
|
$this->logger = new NullLogger();
|
|
$this->loadHtml();
|
|
}
|
|
|
|
public function setLogger(LoggerInterface $logger)
|
|
{
|
|
$this->logger = $logger;
|
|
}
|
|
|
|
/**
|
|
* Get article title element.
|
|
*
|
|
* @return \DOMElement
|
|
*/
|
|
public function getTitle()
|
|
{
|
|
return $this->articleTitle;
|
|
}
|
|
|
|
/**
|
|
* Get article content element.
|
|
*
|
|
* @return \DOMElement
|
|
*/
|
|
public function getContent()
|
|
{
|
|
return $this->articleContent;
|
|
}
|
|
|
|
/**
|
|
* Add pre filter for raw input HTML processing.
|
|
*
|
|
* @param string $filter RegExp for replace
|
|
* @param string $replacer Replacer
|
|
*/
|
|
public function addPreFilter($filter, $replacer = '')
|
|
{
|
|
$this->pre_filters[$filter] = $replacer;
|
|
}
|
|
|
|
/**
|
|
* Add post filter for raw output HTML processing.
|
|
*
|
|
* @param string $filter RegExp for replace
|
|
* @param string $replacer Replacer
|
|
*/
|
|
public function addPostFilter($filter, $replacer = '')
|
|
{
|
|
$this->post_filters[$filter] = $replacer;
|
|
}
|
|
|
|
/**
|
|
* Runs readability.
|
|
*
|
|
* Workflow:
|
|
* 1. Prep the document by removing script tags, css, etc.
|
|
* 2. Build readability's DOM tree.
|
|
* 3. Grab the article content from the current dom tree.
|
|
* 4. Replace the current DOM tree with the new one.
|
|
* 5. Read peacefully.
|
|
*
|
|
* @return bool true if we found content, false otherwise
|
|
*/
|
|
public function init()
|
|
{
|
|
if (!isset($this->dom->documentElement)) {
|
|
return false;
|
|
}
|
|
|
|
// Assume successful outcome
|
|
$this->success = true;
|
|
$bodyElems = $this->dom->getElementsByTagName('body');
|
|
|
|
// WTF multiple body nodes?
|
|
if (null === $this->bodyCache) {
|
|
$this->bodyCache = '';
|
|
foreach ($bodyElems as $bodyNode) {
|
|
$this->bodyCache .= trim($bodyNode->getInnerHTML());
|
|
}
|
|
}
|
|
|
|
if ($bodyElems->length > 0 && null === $this->body) {
|
|
$this->body = $bodyElems->item(0);
|
|
}
|
|
|
|
$this->prepDocument();
|
|
|
|
// Build readability's DOM tree.
|
|
$overlay = $this->dom->createElement('div');
|
|
$innerDiv = $this->dom->createElement('div');
|
|
$articleTitle = $this->getArticleTitle();
|
|
$articleContent = $this->grabArticle();
|
|
|
|
if (!$articleContent) {
|
|
$this->success = false;
|
|
$articleContent = $this->dom->createElement('div');
|
|
$articleContent->setAttribute('class', 'readability-content');
|
|
$articleContent->setInnerHtml('<p>Sorry, Readability was unable to parse this page for content.</p>');
|
|
}
|
|
|
|
$overlay->setAttribute('class', 'readOverlay');
|
|
$innerDiv->setAttribute('class', 'readInner');
|
|
|
|
// Glue the structure of our document together.
|
|
$innerDiv->appendChild($articleTitle);
|
|
$innerDiv->appendChild($articleContent);
|
|
$overlay->appendChild($innerDiv);
|
|
|
|
// without tidy the body can (sometimes) be wiped, so re-create it
|
|
if (false === isset($this->body->childNodes)) {
|
|
$this->body = $this->dom->createElement('body');
|
|
}
|
|
|
|
// Clear the old HTML, insert the new content.
|
|
$this->body->setInnerHtml('');
|
|
$this->body->appendChild($overlay);
|
|
$this->body->removeAttribute('style');
|
|
$this->postProcessContent($articleContent);
|
|
|
|
// Set title and content instance variables.
|
|
$this->articleTitle = $articleTitle;
|
|
$this->articleContent = $articleContent;
|
|
|
|
return $this->success;
|
|
}
|
|
|
|
/**
|
|
* Run any post-process modifications to article content as necessary.
|
|
*/
|
|
public function postProcessContent(\DOMElement $articleContent)
|
|
{
|
|
if ($this->convertLinksToFootnotes && !preg_match('/\bwiki/', $this->url)) {
|
|
$this->addFootnotes($articleContent);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* For easier reading, convert this document to have footnotes at the bottom rather than inline links.
|
|
*
|
|
* @see http://www.roughtype.com/archives/2010/05/experiments_in.php
|
|
*/
|
|
public function addFootnotes(\DOMElement $articleContent)
|
|
{
|
|
$footnotesWrapper = $this->dom->createElement('footer');
|
|
$footnotesWrapper->setAttribute('class', 'readability-footnotes');
|
|
$footnotesWrapper->setInnerHtml('<h3>References</h3>');
|
|
$articleFootnotes = $this->dom->createElement('ol');
|
|
$articleFootnotes->setAttribute('class', 'readability-footnotes-list');
|
|
$footnotesWrapper->appendChild($articleFootnotes);
|
|
$articleLinks = $articleContent->getElementsByTagName('a');
|
|
$linkCount = 0;
|
|
|
|
for ($i = 0; $i < $articleLinks->length; ++$i) {
|
|
$articleLink = $articleLinks->item($i);
|
|
$footnoteLink = $articleLink->cloneNode(true);
|
|
$refLink = $this->dom->createElement('a');
|
|
$footnote = $this->dom->createElement('li');
|
|
$linkDomain = @parse_url($footnoteLink->getAttribute('href'), \PHP_URL_HOST);
|
|
if (!$linkDomain && isset($this->url)) {
|
|
$linkDomain = @parse_url($this->url, \PHP_URL_HOST);
|
|
}
|
|
|
|
$linkText = $this->getInnerText($articleLink);
|
|
if ((false !== strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote')) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
|
|
continue;
|
|
}
|
|
|
|
++$linkCount;
|
|
|
|
// Add a superscript reference after the article link.
|
|
$refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
|
|
$refLink->setInnerHtml('<small><sup>[' . $linkCount . ']</sup></small>');
|
|
$refLink->setAttribute('class', 'readability-DoNotFootnote');
|
|
$refLink->setAttribute('style', 'color: inherit;');
|
|
|
|
if ($articleLink->parentNode->lastChild->isSameNode($articleLink)) {
|
|
$articleLink->parentNode->appendChild($refLink);
|
|
} else {
|
|
$articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
|
|
}
|
|
|
|
$articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
|
|
$articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
|
|
$footnote->setInnerHtml('<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ');
|
|
$footnoteLink->setInnerHtml(('' !== $footnoteLink->getAttribute('title') ? $footnoteLink->getAttribute('title') : $linkText));
|
|
$footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
|
|
$footnote->appendChild($footnoteLink);
|
|
|
|
if ($linkDomain) {
|
|
$footnote->setInnerHtml($footnote->getInnerHTML() . '<small> (' . $linkDomain . ')</small>');
|
|
}
|
|
$articleFootnotes->appendChild($footnote);
|
|
}
|
|
|
|
if ($linkCount > 0) {
|
|
$articleContent->appendChild($footnotesWrapper);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Prepare the article node for display. Clean out any inline styles,
|
|
* iframes, forms, strip extraneous <p> tags, etc.
|
|
*/
|
|
public function prepArticle(\DOMNode $articleContent)
|
|
{
|
|
if (!$articleContent instanceof \DOMElement) {
|
|
return;
|
|
}
|
|
|
|
$this->logger->debug($this->lightClean ? 'Light clean enabled.' : 'Standard clean enabled.');
|
|
|
|
$this->cleanStyles($articleContent);
|
|
$this->killBreaks($articleContent);
|
|
|
|
$xpath = new \DOMXPath($articleContent->ownerDocument);
|
|
|
|
if ($this->revertForcedParagraphElements) {
|
|
/*
|
|
* Reverts P elements with class 'readability-styled' to text nodes:
|
|
* which is what they were before.
|
|
*/
|
|
$elems = $xpath->query('.//p[@data-readability-styled]', $articleContent);
|
|
for ($i = $elems->length - 1; $i >= 0; --$i) {
|
|
$e = $elems->item($i);
|
|
$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
|
|
}
|
|
}
|
|
|
|
// Remove service data-candidate attribute.
|
|
$elems = $xpath->query('.//*[@data-candidate]', $articleContent);
|
|
for ($i = $elems->length - 1; $i >= 0; --$i) {
|
|
$elems->item($i)->removeAttribute('data-candidate');
|
|
}
|
|
|
|
// Clean out junk from the article content.
|
|
$this->clean($articleContent, 'input');
|
|
$this->clean($articleContent, 'button');
|
|
$this->clean($articleContent, 'nav');
|
|
$this->clean($articleContent, 'object');
|
|
$this->clean($articleContent, 'iframe');
|
|
$this->clean($articleContent, 'canvas');
|
|
$this->clean($articleContent, 'h1');
|
|
|
|
/*
|
|
* If there is only one h2, they are probably using it as a main header, so remove it since we
|
|
* already have a header.
|
|
*/
|
|
$h2s = $articleContent->getElementsByTagName('h2');
|
|
if (1 === $h2s->length && mb_strlen($this->getInnerText($h2s->item(0), true, true)) < 100) {
|
|
$this->clean($articleContent, 'h2');
|
|
}
|
|
|
|
$this->cleanHeaders($articleContent);
|
|
|
|
// Do these last as the previous stuff may have removed junk that will affect these.
|
|
$this->cleanConditionally($articleContent, 'form');
|
|
$this->cleanConditionally($articleContent, 'table');
|
|
$this->cleanConditionally($articleContent, 'ul');
|
|
$this->cleanConditionally($articleContent, 'div');
|
|
|
|
// Remove extra paragraphs.
|
|
$articleParagraphs = $articleContent->getElementsByTagName('p');
|
|
|
|
for ($i = $articleParagraphs->length - 1; $i >= 0; --$i) {
|
|
$item = $articleParagraphs->item($i);
|
|
|
|
$imgCount = $item->getElementsByTagName('img')->length;
|
|
$embedCount = $item->getElementsByTagName('embed')->length;
|
|
$objectCount = $item->getElementsByTagName('object')->length;
|
|
$videoCount = $item->getElementsByTagName('video')->length;
|
|
$audioCount = $item->getElementsByTagName('audio')->length;
|
|
$iframeCount = $item->getElementsByTagName('iframe')->length;
|
|
|
|
if (0 === $iframeCount && 0 === $imgCount && 0 === $embedCount && 0 === $objectCount && 0 === $videoCount && 0 === $audioCount && 0 === mb_strlen(preg_replace('/\s+/is', '', $this->getInnerText($item, false, false)))) {
|
|
$item->parentNode->removeChild($item);
|
|
}
|
|
|
|
// add extra text to iframe tag to avoid an auto-closing iframe and then break the html code
|
|
if ($iframeCount) {
|
|
$iframe = $item->getElementsByTagName('iframe');
|
|
$iframe->item(0)->nodeValue = ' ';
|
|
|
|
$item->parentNode->replaceChild($iframe->item(0), $item);
|
|
}
|
|
}
|
|
|
|
if (!$this->flagIsActive(self::FLAG_DISABLE_POSTFILTER)) {
|
|
try {
|
|
foreach ($this->post_filters as $search => $replace) {
|
|
$articleContent->setInnerHtml(preg_replace($search, $replace, $articleContent->getInnerHTML()));
|
|
}
|
|
unset($search, $replace);
|
|
} catch (\Exception $e) {
|
|
$this->logger->error('Cleaning output HTML failed. Ignoring: ' . $e->getMessage());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the inner text of a node.
|
|
* This also strips out any excess whitespace to be found.
|
|
*
|
|
* @param \DOMElement $e
|
|
* @param bool $normalizeSpaces (default: true)
|
|
* @param bool $flattenLines (default: false)
|
|
*
|
|
* @return string
|
|
*/
|
|
public function getInnerText($e, $normalizeSpaces = true, $flattenLines = false)
|
|
{
|
|
if (null === $e || !isset($e->textContent) || '' === $e->textContent) {
|
|
return '';
|
|
}
|
|
|
|
$textContent = trim($e->textContent);
|
|
|
|
if ($flattenLines) {
|
|
$textContent = mb_ereg_replace('(?:[\r\n](?:\s| )*)+', '', $textContent);
|
|
} elseif ($normalizeSpaces) {
|
|
$textContent = mb_ereg_replace('\s\s+', ' ', $textContent);
|
|
}
|
|
|
|
return $textContent;
|
|
}
|
|
|
|
/**
|
|
* Remove the style attribute on every $e and under.
|
|
*
|
|
* @param \DOMElement $e
|
|
*/
|
|
public function cleanStyles($e)
|
|
{
|
|
if (!\is_object($e)) {
|
|
return;
|
|
}
|
|
|
|
$elems = $e->getElementsByTagName('*');
|
|
|
|
foreach ($elems as $elem) {
|
|
$elem->removeAttribute('style');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get comma number for a given text.
|
|
*
|
|
* @param string $text
|
|
*
|
|
* @return int
|
|
*/
|
|
public function getCommaCount($text)
|
|
{
|
|
return substr_count($text, ',');
|
|
}
|
|
|
|
/**
|
|
* Get words number for a given text if words separated by a space.
|
|
* Input string should be normalized.
|
|
*
|
|
* @param string $text
|
|
*
|
|
* @return int
|
|
*/
|
|
public function getWordCount($text)
|
|
{
|
|
return substr_count($text, ' ');
|
|
}
|
|
|
|
/**
|
|
* Get the density of links as a percentage of the content
|
|
* This is the amount of text that is inside a link divided by the total text in the node.
|
|
* Can exclude external references to differentiate between simple text and menus/infoblocks.
|
|
*
|
|
* @param bool $excludeExternal
|
|
*
|
|
* @return int
|
|
*/
|
|
public function getLinkDensity(\DOMElement $e, $excludeExternal = false)
|
|
{
|
|
$links = $e->getElementsByTagName('a');
|
|
$textLength = mb_strlen($this->getInnerText($e, true, true));
|
|
$linkLength = 0;
|
|
|
|
for ($dRe = $this->domainRegExp, $i = 0, $il = $links->length; $i < $il; ++$i) {
|
|
if ($excludeExternal && $dRe && !preg_match($dRe, $links->item($i)->getAttribute('href'))) {
|
|
continue;
|
|
}
|
|
$linkLength += mb_strlen($this->getInnerText($links->item($i)));
|
|
}
|
|
|
|
if ($textLength > 0 && $linkLength > 0) {
|
|
return $linkLength / $textLength;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Get an element relative weight.
|
|
*
|
|
* @return int
|
|
*/
|
|
public function getWeight(\DOMElement $e)
|
|
{
|
|
if (!$this->flagIsActive(self::FLAG_WEIGHT_ATTRIBUTES)) {
|
|
return 0;
|
|
}
|
|
|
|
$weight = 0;
|
|
// Look for a special classname
|
|
$weight += $this->weightAttribute($e, 'class');
|
|
// Look for a special ID
|
|
$weight += $this->weightAttribute($e, 'id');
|
|
|
|
return $weight;
|
|
}
|
|
|
|
/**
|
|
* Remove extraneous break tags from a node.
|
|
*/
|
|
public function killBreaks(\DOMElement $node)
|
|
{
|
|
$html = $node->getInnerHTML();
|
|
$html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
|
|
$node->setInnerHtml($html);
|
|
}
|
|
|
|
/**
|
|
* Clean a node of all elements of type "tag".
|
|
* (Unless it's a youtube/vimeo video. People love movies.).
|
|
*
|
|
* Updated 2012-09-18 to preserve youtube/vimeo iframes
|
|
*
|
|
* @param string $tag
|
|
*/
|
|
public function clean(\DOMElement $e, $tag)
|
|
{
|
|
$targetList = $e->getElementsByTagName($tag);
|
|
$isEmbed = ('audio' === $tag || 'video' === $tag || 'iframe' === $tag || 'object' === $tag || 'embed' === $tag);
|
|
|
|
for ($y = $targetList->length - 1; $y >= 0; --$y) {
|
|
// Allow youtube and vimeo videos through as people usually want to see those.
|
|
$currentItem = $targetList->item($y);
|
|
|
|
if ($isEmbed) {
|
|
$attributeValues = $currentItem->getAttribute('src') . ' ' . $currentItem->getAttribute('href');
|
|
|
|
// First, check the elements attributes to see if any of them contain known media hosts
|
|
if (preg_match($this->regexps['media'], $attributeValues)) {
|
|
continue;
|
|
}
|
|
|
|
// Then check the elements inside this element for the same.
|
|
if (preg_match($this->regexps['media'], $targetList->item($y)->getInnerHTML())) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
$currentItem->parentNode->removeChild($currentItem);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clean an element of all tags of type "tag" if they look fishy.
|
|
* "Fishy" is an algorithm based on content length, classnames,
|
|
* link density, number of images & embeds, etc.
|
|
*
|
|
* @param string $tag
|
|
*/
|
|
public function cleanConditionally(\DOMElement $e, $tag)
|
|
{
|
|
if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
|
|
return;
|
|
}
|
|
|
|
$tagsList = $e->getElementsByTagName($tag);
|
|
$curTagsLength = $tagsList->length;
|
|
|
|
/*
|
|
* Gather counts for other typical elements embedded within.
|
|
* Traverse backwards so we can remove nodes at the same time without effecting the traversal.
|
|
*
|
|
* TODO: Consider taking into account original contentScore here.
|
|
*/
|
|
for ($i = $curTagsLength - 1; $i >= 0; --$i) {
|
|
$node = $tagsList->item($i);
|
|
$weight = $this->getWeight($node);
|
|
$contentScore = ($node->hasAttribute('readability')) ? (int) $node->getAttribute('readability') : 0;
|
|
$this->logger->debug('Start conditional cleaning of ' . $node->getNodePath() . ' (class=' . $node->getAttribute('class') . '; id=' . $node->getAttribute('id') . ')' . (($node->hasAttribute('readability')) ? (' with score ' . $node->getAttribute('readability')) : ''));
|
|
|
|
if ($weight + $contentScore < 0) {
|
|
$this->logger->debug('Removing...');
|
|
$node->parentNode->removeChild($node);
|
|
} elseif ($this->getCommaCount($this->getInnerText($node)) < self::MIN_COMMAS_IN_PARAGRAPH) {
|
|
/*
|
|
* If there are not very many commas, and the number of
|
|
* non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
|
|
*/
|
|
$p = $node->getElementsByTagName('p')->length;
|
|
$img = $node->getElementsByTagName('img')->length;
|
|
$li = $node->getElementsByTagName('li')->length - 100;
|
|
$input = $node->getElementsByTagName('input')->length;
|
|
$a = $node->getElementsByTagName('a')->length;
|
|
$embedCount = 0;
|
|
$embeds = $node->getElementsByTagName('embed');
|
|
|
|
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
|
|
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
|
|
++$embedCount;
|
|
}
|
|
}
|
|
|
|
$embeds = $node->getElementsByTagName('iframe');
|
|
for ($ei = 0, $il = $embeds->length; $ei < $il; ++$ei) {
|
|
if (preg_match($this->regexps['media'], $embeds->item($ei)->getAttribute('src'))) {
|
|
++$embedCount;
|
|
}
|
|
}
|
|
|
|
$linkDensity = $this->getLinkDensity($node, true);
|
|
$contentLength = mb_strlen($this->getInnerText($node));
|
|
$toRemove = false;
|
|
|
|
if ($this->lightClean) {
|
|
if ($li > $p && 'ul' !== $tag && 'ol' !== $tag) {
|
|
$this->logger->debug(' too many <li> elements, and parent is not <ul> or <ol>');
|
|
$toRemove = true;
|
|
} elseif ($input > floor($p / 3)) {
|
|
$this->logger->debug(' too many <input> elements');
|
|
$toRemove = true;
|
|
} elseif ($contentLength < 6 && (0 === $embedCount && (0 === $img || $img > 2))) {
|
|
$this->logger->debug(' content length less than 6 chars, 0 embeds and either 0 images or more than 2 images');
|
|
$toRemove = true;
|
|
} elseif ($weight < 25 && $linkDensity > 0.25) {
|
|
$this->logger->debug(' weight is ' . $weight . ' < 25 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.25');
|
|
$toRemove = true;
|
|
} elseif ($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
|
|
$this->logger->debug(' more than 2 links and weight is ' . $weight . ' > 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
|
|
$toRemove = true;
|
|
} elseif ($embedCount > 3) {
|
|
$this->logger->debug(' more than 3 embeds');
|
|
$toRemove = true;
|
|
}
|
|
} else {
|
|
if ($img > $p) {
|
|
$this->logger->debug(' more image elements than paragraph elements');
|
|
$toRemove = true;
|
|
} elseif ($li > $p && 'ul' !== $tag && 'ol' !== $tag) {
|
|
$this->logger->debug(' too many <li> elements, and parent is not <ul> or <ol>');
|
|
$toRemove = true;
|
|
} elseif ($input > floor($p / 3)) {
|
|
$this->logger->debug(' too many <input> elements');
|
|
$toRemove = true;
|
|
} elseif ($contentLength < 10 && (0 === $img || $img > 2)) {
|
|
$this->logger->debug(' content length less than 10 chars and 0 images, or more than 2 images');
|
|
$toRemove = true;
|
|
} elseif ($weight < 25 && $linkDensity > 0.2) {
|
|
$this->logger->debug(' weight is ' . $weight . ' lower than 0 and link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.2');
|
|
$toRemove = true;
|
|
} elseif ($weight >= 25 && $linkDensity > 0.5) {
|
|
$this->logger->debug(' weight above 25 but link density is ' . sprintf('%.2f', $linkDensity) . ' > 0.5');
|
|
$toRemove = true;
|
|
} elseif ((1 === $embedCount && $contentLength < 75) || $embedCount > 1) {
|
|
$this->logger->debug(' 1 embed and content length smaller than 75 chars, or more than one embed');
|
|
$toRemove = true;
|
|
}
|
|
}
|
|
|
|
if ($toRemove) {
|
|
$this->logger->debug('Removing...');
|
|
$node->parentNode->removeChild($node);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clean out spurious headers from an Element. Checks things like classnames and link density.
|
|
*/
|
|
public function cleanHeaders(\DOMElement $e)
|
|
{
|
|
for ($headerIndex = 1; $headerIndex < 3; ++$headerIndex) {
|
|
$headers = $e->getElementsByTagName('h' . $headerIndex);
|
|
|
|
for ($i = $headers->length - 1; $i >= 0; --$i) {
|
|
if ($this->getWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
|
|
$headers->item($i)->parentNode->removeChild($headers->item($i));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if the given flag is active.
|
|
*
|
|
* @param int $flag
|
|
*
|
|
* @return bool
|
|
*/
|
|
public function flagIsActive($flag)
|
|
{
|
|
return ($this->flags & $flag) > 0;
|
|
}
|
|
|
|
/**
|
|
* Add a flag.
|
|
*
|
|
* @param int $flag
|
|
*/
|
|
public function addFlag($flag)
|
|
{
|
|
$this->flags = $this->flags | $flag;
|
|
}
|
|
|
|
/**
|
|
* Remove a flag.
|
|
*
|
|
* @param int $flag
|
|
*/
|
|
public function removeFlag($flag)
|
|
{
|
|
$this->flags = $this->flags & ~$flag;
|
|
}
|
|
|
|
/**
|
|
* Debug.
|
|
*
|
|
* @deprecated use $this->logger->debug() instead
|
|
* @codeCoverageIgnore
|
|
*/
|
|
protected function dbg($msg)
|
|
{
|
|
$this->logger->debug($msg);
|
|
}
|
|
|
|
/**
|
|
* Dump debug info.
|
|
*
|
|
* @deprecated since Monolog gather log, we don't need it
|
|
* @codeCoverageIgnore
|
|
*/
|
|
protected function dump_dbg()
|
|
{
|
|
}
|
|
|
|
/**
|
|
* Get the article title as an H1.
|
|
*
|
|
* @return \DOMElement
|
|
*/
|
|
protected function getArticleTitle()
|
|
{
|
|
try {
|
|
$curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
|
|
} catch (\Exception $e) {
|
|
$curTitle = '';
|
|
$origTitle = '';
|
|
}
|
|
|
|
if (preg_match('/ [\|\-] /', $curTitle)) {
|
|
$curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
|
|
if (\count(explode(' ', $curTitle)) < 3) {
|
|
$curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
|
|
}
|
|
} elseif (false !== strpos($curTitle, ': ')) {
|
|
$curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
|
|
if (\count(explode(' ', $curTitle)) < 3) {
|
|
$curTitle = preg_replace('/[^:]*[:](.*)/i', '$1', $origTitle);
|
|
}
|
|
} elseif (mb_strlen($curTitle) > 150 || mb_strlen($curTitle) < 15) {
|
|
$hOnes = $this->dom->getElementsByTagName('h1');
|
|
if (1 === $hOnes->length) {
|
|
$curTitle = $this->getInnerText($hOnes->item(0));
|
|
}
|
|
}
|
|
|
|
$curTitle = trim($curTitle);
|
|
if (\count(explode(' ', $curTitle)) <= 4) {
|
|
$curTitle = $origTitle;
|
|
}
|
|
|
|
$articleTitle = $this->dom->createElement('h1');
|
|
$articleTitle->setInnerHtml($curTitle);
|
|
|
|
return $articleTitle;
|
|
}
|
|
|
|
/**
|
|
* Prepare the HTML document for readability to scrape it.
|
|
* This includes things like stripping javascript, CSS, and handling terrible markup.
|
|
*/
|
|
protected function prepDocument()
|
|
{
|
|
/*
|
|
* In some cases a body element can't be found (if the HTML is totally hosed for example)
|
|
* so we create a new body node and append it to the document.
|
|
*/
|
|
if (null === $this->body) {
|
|
$this->body = $this->dom->createElement('body');
|
|
$this->dom->documentElement->appendChild($this->body);
|
|
}
|
|
|
|
$this->body->setAttribute('class', 'readabilityBody');
|
|
|
|
// Remove all style tags in head.
|
|
$styleTags = $this->dom->getElementsByTagName('style');
|
|
for ($i = $styleTags->length - 1; $i >= 0; --$i) {
|
|
$styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
|
|
}
|
|
|
|
$linkTags = $this->dom->getElementsByTagName('link');
|
|
for ($i = $linkTags->length - 1; $i >= 0; --$i) {
|
|
$linkTags->item($i)->parentNode->removeChild($linkTags->item($i));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialize a node with the readability object. Also checks the
|
|
* className/id for special names to add to its score.
|
|
*/
|
|
protected function initializeNode(\DOMElement $node)
|
|
{
|
|
if (!isset($node->tagName)) {
|
|
return;
|
|
}
|
|
|
|
$readability = $this->dom->createAttribute('readability');
|
|
// this is our contentScore
|
|
$readability->value = 0;
|
|
$node->setAttributeNode($readability);
|
|
|
|
// using strtoupper just in case
|
|
switch (strtoupper($node->tagName)) {
|
|
case 'ARTICLE':
|
|
$readability->value += 15;
|
|
// no break
|
|
case 'DIV':
|
|
$readability->value += 5;
|
|
break;
|
|
case 'PRE':
|
|
case 'CODE':
|
|
case 'TD':
|
|
case 'BLOCKQUOTE':
|
|
case 'FIGURE':
|
|
$readability->value += 3;
|
|
break;
|
|
case 'SECTION':
|
|
// often misused
|
|
// $readability->value += 2;
|
|
break;
|
|
case 'OL':
|
|
case 'UL':
|
|
case 'DL':
|
|
case 'DD':
|
|
case 'DT':
|
|
case 'LI':
|
|
$readability->value -= 2 * round($this->getLinkDensity($node), 0, \PHP_ROUND_HALF_UP);
|
|
break;
|
|
case 'ASIDE':
|
|
case 'FOOTER':
|
|
case 'HEADER':
|
|
case 'ADDRESS':
|
|
case 'FORM':
|
|
case 'BUTTON':
|
|
case 'TEXTAREA':
|
|
case 'INPUT':
|
|
case 'NAV':
|
|
$readability->value -= 3;
|
|
break;
|
|
case 'H1':
|
|
case 'H2':
|
|
case 'H3':
|
|
case 'H4':
|
|
case 'H5':
|
|
case 'H6':
|
|
case 'TH':
|
|
case 'HGROUP':
|
|
$readability->value -= 5;
|
|
break;
|
|
}
|
|
|
|
$readability->value += $this->getWeight($node);
|
|
}
|
|
|
|
/**
|
|
* Using a variety of metrics (content score, classname, element types), find the content that is
|
|
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
|
|
*
|
|
* @param \DOMElement $page
|
|
*
|
|
* @return \DOMElement|false
|
|
*/
|
|
protected function grabArticle(\DOMElement $page = null)
|
|
{
|
|
if (!$page) {
|
|
$page = $this->dom;
|
|
}
|
|
|
|
$xpath = null;
|
|
$nodesToScore = [];
|
|
|
|
if ($page instanceof \DOMDocument && isset($page->documentElement)) {
|
|
$xpath = new \DOMXPath($page);
|
|
}
|
|
|
|
$allElements = $page->getElementsByTagName('*');
|
|
|
|
for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); ++$nodeIndex) {
|
|
$tagName = $node->tagName;
|
|
|
|
$nodeContent = $node->getInnerHTML();
|
|
if (empty($nodeContent)) {
|
|
$this->logger->debug('Skipping empty node');
|
|
continue;
|
|
}
|
|
|
|
// Some well known site uses sections as paragraphs.
|
|
if (0 === strcasecmp($tagName, 'p') || 0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'pre') || 0 === strcasecmp($tagName, 'section')) {
|
|
$nodesToScore[] = $node;
|
|
}
|
|
|
|
// Turn divs into P tags where they have been used inappropriately
|
|
// (as in, where they contain no other block level elements).
|
|
if (0 === strcasecmp($tagName, 'div') || 0 === strcasecmp($tagName, 'article') || 0 === strcasecmp($tagName, 'section')) {
|
|
if (!preg_match($this->regexps['divToPElements'], $nodeContent)) {
|
|
$newNode = $this->dom->createElement('p');
|
|
|
|
try {
|
|
$newNode->setInnerHtml($nodeContent);
|
|
|
|
$node->parentNode->replaceChild($newNode, $node);
|
|
--$nodeIndex;
|
|
$nodesToScore[] = $newNode;
|
|
} catch (\Exception $e) {
|
|
$this->logger->error('Could not alter div/article to p, reverting back to div: ' . $e->getMessage());
|
|
}
|
|
} else {
|
|
// Will change these P elements back to text nodes after processing.
|
|
for ($i = 0, $il = $node->childNodes->length; $i < $il; ++$i) {
|
|
$childNode = $node->childNodes->item($i);
|
|
|
|
// it looks like sometimes the loop is going too far and we are retrieving a non-existant child
|
|
if (null === $childNode) {
|
|
continue;
|
|
}
|
|
|
|
// executable tags (<?php or <?xml) warning
|
|
if (\is_object($childNode) && 'DOMProcessingInstruction' === \get_class($childNode)) {
|
|
$childNode->parentNode->removeChild($childNode);
|
|
|
|
continue;
|
|
}
|
|
|
|
if (\XML_TEXT_NODE === $childNode->nodeType) {
|
|
$p = $this->dom->createElement('p');
|
|
$p->setInnerHtml($childNode->nodeValue);
|
|
$p->setAttribute('data-readability-styled', 'true');
|
|
$childNode->parentNode->replaceChild($p, $childNode);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Loop through all paragraphs, and assign a score to them based on how content-y they look.
|
|
* Then add their score to their parent node.
|
|
*
|
|
* A score is determined by things like number of commas, class names, etc.
|
|
* Maybe eventually link density.
|
|
*/
|
|
for ($pt = 0, $scored = \count($nodesToScore); $pt < $scored; ++$pt) {
|
|
$parentNode = $nodesToScore[$pt]->parentNode;
|
|
|
|
// No parent node? Move on...
|
|
if (!$parentNode) {
|
|
continue;
|
|
}
|
|
|
|
$grandParentNode = $parentNode->parentNode instanceof \DOMElement ? $parentNode->parentNode : null;
|
|
$innerText = $this->getInnerText($nodesToScore[$pt]);
|
|
|
|
// If this paragraph is less than MIN_PARAGRAPH_LENGTH (default:20) characters, don't even count it.
|
|
if (mb_strlen($innerText) < self::MIN_PARAGRAPH_LENGTH) {
|
|
continue;
|
|
}
|
|
|
|
// Initialize readability data for the parent.
|
|
if (!$parentNode->hasAttribute('readability')) {
|
|
$this->initializeNode($parentNode);
|
|
$parentNode->setAttribute('data-candidate', 'true');
|
|
}
|
|
|
|
// Initialize readability data for the grandparent.
|
|
if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) {
|
|
$this->initializeNode($grandParentNode);
|
|
$grandParentNode->setAttribute('data-candidate', 'true');
|
|
}
|
|
// Add a point for the paragraph itself as a base.
|
|
$contentScore = 1;
|
|
// Add points for any commas within this paragraph.
|
|
$contentScore += $this->getCommaCount($innerText);
|
|
// For every SCORE_CHARS_IN_PARAGRAPH (default:100) characters in this paragraph, add another point. Up to 3 points.
|
|
$contentScore += min(floor(mb_strlen($innerText) / self::SCORE_CHARS_IN_PARAGRAPH), 3);
|
|
// For every SCORE_WORDS_IN_PARAGRAPH (default:20) words in this paragraph, add another point. Up to 3 points.
|
|
$contentScore += min(floor($this->getWordCount($innerText) / self::SCORE_WORDS_IN_PARAGRAPH), 3);
|
|
/* TEST: For every positive/negative parent tag, add/substract half point. Up to 3 points. *\/
|
|
$up = $nodesToScore[$pt];
|
|
$score = 0;
|
|
while ($up->parentNode instanceof \DOMElement) {
|
|
$up = $up->parentNode;
|
|
if (preg_match($this->regexps['positive'], $up->getAttribute('class') . ' ' . $up->getAttribute('id'))) {
|
|
$score += 0.5;
|
|
} elseif (preg_match($this->regexps['negative'], $up->getAttribute('class') . ' ' . $up->getAttribute('id'))) {
|
|
$score -= 0.5;
|
|
}
|
|
}
|
|
$score = floor($score);
|
|
$contentScore += max(min($score, 3), -3);/**/
|
|
|
|
// Add the score to the parent. The grandparent gets half.
|
|
$parentNode->getAttributeNode('readability')->value += $contentScore;
|
|
if ($grandParentNode) {
|
|
$grandParentNode->getAttributeNode('readability')->value += round($contentScore / self::GRANDPARENT_SCORE_DIVISOR);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Node prepping: trash nodes that look cruddy (like ones with the class name "comment", etc).
|
|
* This is faster to do before scoring but safer after.
|
|
*/
|
|
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS) && $xpath) {
|
|
$candidates = $xpath->query('.//*[(self::footer and count(//footer)<2) or (self::aside and count(//aside)<2)]', $page->documentElement);
|
|
|
|
for ($c = $candidates->length - 1; $c >= 0; --$c) {
|
|
$node = $candidates->item($c);
|
|
// node should be readable but not inside of an article otherwise it's probably non-readable block
|
|
if ($node->hasAttribute('readability') && (int) $node->getAttributeNode('readability')->value < 40 && ($node->parentNode ? 0 !== strcasecmp($node->parentNode->tagName, 'article') : true)) {
|
|
$this->logger->debug('Removing unlikely candidate (using note) ' . $node->getNodePath() . ' by "' . $node->tagName . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
|
|
$node->parentNode->removeChild($node);
|
|
}
|
|
}
|
|
|
|
$candidates = $xpath->query('.//*[not(self::body) and (@class or @id or @style) and ((number(@readability) < 40) or not(@readability))]', $page->documentElement);
|
|
|
|
for ($c = $candidates->length - 1; $c >= 0; --$c) {
|
|
$node = $candidates->item($c);
|
|
|
|
// Remove unlikely candidates
|
|
$unlikelyMatchString = $node->getAttribute('class') . ' ' . $node->getAttribute('id') . ' ' . $node->getAttribute('style');
|
|
|
|
if (mb_strlen($unlikelyMatchString) > 3 && // don't process "empty" strings
|
|
preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
|
|
!preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString)
|
|
) {
|
|
$this->logger->debug('Removing unlikely candidate (using conf) ' . $node->getNodePath() . ' by "' . $unlikelyMatchString . '" with readability ' . ($node->hasAttribute('readability') ? (int) $node->getAttributeNode('readability')->value : 0));
|
|
$node->parentNode->removeChild($node);
|
|
--$nodeIndex;
|
|
}
|
|
}
|
|
unset($candidates);
|
|
}
|
|
|
|
/*
|
|
* After we've calculated scores, loop through all of the possible candidate nodes we found
|
|
* and find the one with the highest score.
|
|
*/
|
|
$topCandidate = null;
|
|
if ($xpath) {
|
|
// Using array of DOMElements after deletion is a path to DOOMElement.
|
|
$candidates = $xpath->query('.//*[@data-candidate]', $page->documentElement);
|
|
|
|
for ($c = $candidates->length - 1; $c >= 0; --$c) {
|
|
$item = $candidates->item($c);
|
|
|
|
// Scale the final candidates score based on link density. Good content should have a
|
|
// relatively small link density (5% or less) and be mostly unaffected by this operation.
|
|
// If not for this we would have used XPath to find maximum @readability.
|
|
$readability = $item->getAttributeNode('readability');
|
|
$readability->value = round($readability->value * (1 - $this->getLinkDensity($item)), 0, \PHP_ROUND_HALF_UP);
|
|
|
|
if (!$topCandidate || $readability->value > (int) $topCandidate->getAttribute('readability')) {
|
|
$this->logger->debug('Candidate: ' . $item->getNodePath() . ' (' . $item->getAttribute('class') . ':' . $item->getAttribute('id') . ') with score ' . $readability->value);
|
|
$topCandidate = $item;
|
|
}
|
|
}
|
|
|
|
unset($candidates);
|
|
}
|
|
|
|
/*
|
|
* If we still have no top candidate, just use the body as a last resort.
|
|
* We also have to copy the body node so it is something we can modify.
|
|
*/
|
|
if (null === $topCandidate || 0 === strcasecmp($topCandidate->tagName, 'body')) {
|
|
$topCandidate = $this->dom->createElement('div');
|
|
|
|
if ($page instanceof \DOMDocument) {
|
|
if (!isset($page->documentElement)) {
|
|
// we don't have a body either? what a mess! :)
|
|
$this->logger->debug('The page has no body!');
|
|
} else {
|
|
$this->logger->debug('Setting body to a raw HTML of original page!');
|
|
$topCandidate->setInnerHtml($page->documentElement->getInnerHTML());
|
|
$page->documentElement->setInnerHtml('');
|
|
$this->reinitBody();
|
|
$page->documentElement->appendChild($topCandidate);
|
|
}
|
|
} else {
|
|
$topCandidate->setInnerHtml($page->getInnerHTML());
|
|
$page->setInnerHtml('');
|
|
$page->appendChild($topCandidate);
|
|
}
|
|
|
|
$this->initializeNode($topCandidate);
|
|
}
|
|
|
|
// Set table as the main node if resulted data is table element.
|
|
$tagName = $topCandidate->tagName;
|
|
if (0 === strcasecmp($tagName, 'td') || 0 === strcasecmp($tagName, 'tr')) {
|
|
$up = $topCandidate;
|
|
|
|
if ($up->parentNode instanceof \DOMElement) {
|
|
$up = $up->parentNode;
|
|
|
|
if (0 === strcasecmp($up->tagName, 'table')) {
|
|
$topCandidate = $up;
|
|
}
|
|
}
|
|
}
|
|
|
|
$this->logger->debug('Top candidate: ' . $topCandidate->getNodePath());
|
|
|
|
/*
|
|
* Now that we have the top candidate, look through its siblings for content that might also be related.
|
|
* Things like preambles, content split by ads that we removed, etc.
|
|
*/
|
|
$articleContent = $this->dom->createElement('div');
|
|
$articleContent->setAttribute('class', 'readability-content');
|
|
$siblingScoreThreshold = max(10, ((int) $topCandidate->getAttribute('readability')) * 0.2);
|
|
$siblingNodes = $topCandidate->parentNode->childNodes;
|
|
|
|
if (null === $siblingNodes) {
|
|
$siblingNodes = new \stdClass();
|
|
$siblingNodes->length = 0;
|
|
}
|
|
|
|
for ($s = 0, $sl = $siblingNodes->length; $s < $sl; ++$s) {
|
|
$siblingNode = $siblingNodes->item($s);
|
|
$siblingNodeName = $siblingNode->nodeName;
|
|
$append = false;
|
|
$this->logger->debug('Looking at sibling node: ' . $siblingNode->getNodePath() . ((\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
|
|
|
|
if ($siblingNode->isSameNode($topCandidate)) {
|
|
$append = true;
|
|
}
|
|
|
|
$contentBonus = 0;
|
|
|
|
// Give a bonus if sibling nodes and top candidates have the same classname.
|
|
if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->getAttribute('class') === $topCandidate->getAttribute('class') && '' !== $topCandidate->getAttribute('class')) {
|
|
$contentBonus += ((int) $topCandidate->getAttribute('readability')) * 0.2;
|
|
}
|
|
|
|
if (\XML_ELEMENT_NODE === $siblingNode->nodeType && $siblingNode->hasAttribute('readability') && (((int) $siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) {
|
|
$append = true;
|
|
}
|
|
|
|
if (0 === strcasecmp($siblingNodeName, 'p')) {
|
|
$linkDensity = $this->getLinkDensity($siblingNode);
|
|
$nodeContent = $this->getInnerText($siblingNode, true, true);
|
|
$nodeLength = mb_strlen($nodeContent);
|
|
|
|
if (($nodeLength > self::MIN_NODE_LENGTH && $linkDensity < self::MAX_LINK_DENSITY)
|
|
|| ($nodeLength < self::MIN_NODE_LENGTH && 0 === $linkDensity && preg_match('/\.( |$)/', $nodeContent))) {
|
|
$append = true;
|
|
}
|
|
}
|
|
|
|
if ($append) {
|
|
$this->logger->debug('Appending node: ' . $siblingNode->getNodePath());
|
|
|
|
if (0 !== strcasecmp($siblingNodeName, 'div') && 0 !== strcasecmp($siblingNodeName, 'p')) {
|
|
// We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident.
|
|
$this->logger->debug('Altering siblingNode "' . $siblingNodeName . '" to "div".');
|
|
$nodeToAppend = $this->dom->createElement('div');
|
|
|
|
try {
|
|
$nodeToAppend->setAttribute('alt', $siblingNodeName);
|
|
$nodeToAppend->setInnerHtml($siblingNode->getInnerHTML());
|
|
} catch (\Exception $e) {
|
|
$this->logger->debug('Could not alter siblingNode "' . $siblingNodeName . '" to "div", reverting to original.');
|
|
$nodeToAppend = $siblingNode;
|
|
--$s;
|
|
--$sl;
|
|
}
|
|
} else {
|
|
$nodeToAppend = $siblingNode;
|
|
--$s;
|
|
--$sl;
|
|
}
|
|
|
|
// To ensure a node does not interfere with readability styles, remove its classnames & ids.
|
|
// Now done via RegExp post_filter.
|
|
//$nodeToAppend->removeAttribute('class');
|
|
//$nodeToAppend->removeAttribute('id');
|
|
// Append sibling and subtract from our list as appending removes a node.
|
|
$articleContent->appendChild($nodeToAppend);
|
|
}
|
|
}
|
|
|
|
unset($xpath);
|
|
|
|
// So we have all of the content that we need. Now we clean it up for presentation.
|
|
$this->prepArticle($articleContent);
|
|
|
|
/*
|
|
* Now that we've gone through the full algorithm, check to see if we got any meaningful content.
|
|
* If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
|
|
* likelihood of finding the content, and the sieve approach gives us a higher likelihood of
|
|
* finding the -right- content.
|
|
*/
|
|
if (mb_strlen($this->getInnerText($articleContent, false)) < self::MIN_ARTICLE_LENGTH) {
|
|
$this->reinitBody();
|
|
|
|
if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
|
|
$this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
|
|
$this->logger->debug('...content is shorter than ' . self::MIN_ARTICLE_LENGTH . " letters, trying not to strip unlikely content.\n");
|
|
|
|
return $this->grabArticle($this->body);
|
|
} elseif ($this->flagIsActive(self::FLAG_WEIGHT_ATTRIBUTES)) {
|
|
$this->removeFlag(self::FLAG_WEIGHT_ATTRIBUTES);
|
|
$this->logger->debug('...content is shorter than ' . self::MIN_ARTICLE_LENGTH . " letters, trying not to weight attributes.\n");
|
|
|
|
return $this->grabArticle($this->body);
|
|
} elseif ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
|
|
$this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
|
|
$this->logger->debug('...content is shorter than ' . self::MIN_ARTICLE_LENGTH . " letters, trying not to clean at all.\n");
|
|
|
|
return $this->grabArticle($this->body);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
return $articleContent;
|
|
}
|
|
|
|
/**
|
|
* Get an element weight by attribute.
|
|
* Uses regular expressions to tell if this element looks good or bad.
|
|
*
|
|
* @param string $attribute
|
|
*
|
|
* @return int
|
|
*/
|
|
protected function weightAttribute(\DOMElement $element, $attribute)
|
|
{
|
|
if (!$element->hasAttribute($attribute)) {
|
|
return 0;
|
|
}
|
|
$weight = 0;
|
|
|
|
// $attributeValue = trim($element->getAttribute('class')." ".$element->getAttribute('id'));
|
|
$attributeValue = trim($element->getAttribute($attribute));
|
|
|
|
if ('' !== $attributeValue) {
|
|
if (preg_match($this->regexps['negative'], $attributeValue)) {
|
|
$weight -= 25;
|
|
}
|
|
if (preg_match($this->regexps['positive'], $attributeValue)) {
|
|
$weight += 25;
|
|
}
|
|
if (preg_match($this->regexps['unlikelyCandidates'], $attributeValue)) {
|
|
$weight -= 5;
|
|
}
|
|
if (preg_match($this->regexps['okMaybeItsACandidate'], $attributeValue)) {
|
|
$weight += 5;
|
|
}
|
|
}
|
|
|
|
return $weight;
|
|
}
|
|
|
|
/**
|
|
* Will recreate previously deleted body property.
|
|
*/
|
|
protected function reinitBody()
|
|
{
|
|
if (!isset($this->body->childNodes)) {
|
|
$this->body = $this->dom->createElement('body');
|
|
$this->body->setInnerHtml($this->bodyCache);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load HTML in a DOMDocument.
|
|
* Apply Pre filters
|
|
* Cleanup HTML using Tidy (or not).
|
|
*
|
|
* @todo This should be called in init() instead of from __construct
|
|
*/
|
|
private function loadHtml()
|
|
{
|
|
$this->original_html = $this->html;
|
|
|
|
$this->logger->debug('Parsing URL: ' . $this->url);
|
|
|
|
if ($this->url) {
|
|
$this->domainRegExp = '/' . strtr(preg_replace('/www\d*\./', '', parse_url($this->url, \PHP_URL_HOST)), ['.' => '\.']) . '/';
|
|
}
|
|
|
|
mb_internal_encoding('UTF-8');
|
|
mb_http_output('UTF-8');
|
|
mb_regex_encoding('UTF-8');
|
|
|
|
// HACK: dirty cleanup to replace some stuff; shouldn't use regexps with HTML but well...
|
|
if (!$this->flagIsActive(self::FLAG_DISABLE_PREFILTER)) {
|
|
foreach ($this->pre_filters as $search => $replace) {
|
|
$this->html = preg_replace($search, $replace, $this->html);
|
|
}
|
|
unset($search, $replace);
|
|
}
|
|
|
|
if ('' === trim($this->html)) {
|
|
$this->html = '<html></html>';
|
|
}
|
|
|
|
/*
|
|
* Use tidy (if it exists).
|
|
* This fixes problems with some sites which would otherwise trouble DOMDocument's HTML parsing.
|
|
* Although sometimes it makes matters worse, which is why there is an option to disable it.
|
|
*/
|
|
if ($this->useTidy) {
|
|
$this->logger->debug('Tidying document');
|
|
|
|
$tidy = tidy_repair_string($this->html, $this->tidy_config, 'UTF8');
|
|
if (false !== $tidy && $this->html !== $tidy) {
|
|
$this->tidied = true;
|
|
$this->html = $tidy;
|
|
$this->html = preg_replace('/[\r\n]+/is', "\n", $this->html);
|
|
}
|
|
unset($tidy);
|
|
}
|
|
|
|
$this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');
|
|
|
|
if ('html5lib' === $this->parser || 'html5' === $this->parser) {
|
|
$this->dom = (new HTML5())->loadHTML($this->html);
|
|
}
|
|
|
|
if ('libxml' === $this->parser) {
|
|
libxml_use_internal_errors(true);
|
|
|
|
$this->dom = new \DOMDocument();
|
|
$this->dom->preserveWhiteSpace = false;
|
|
$this->dom->loadHTML($this->html, \LIBXML_NOBLANKS | \LIBXML_COMPACT | \LIBXML_NOERROR);
|
|
|
|
libxml_use_internal_errors(false);
|
|
}
|
|
|
|
$this->dom->registerNodeClass('DOMElement', 'Readability\JSLikeHTMLElement');
|
|
}
|
|
}
|