Compare commits

..
2 Commits
Author SHA1 Message Date
Jérémy BenoistandGitHub 5638357306 Merge pull request #74 from Kdecherf/fix/pre-filters-failure 2022-06-13 06:15:24 +02:00
Kevin Decherf 651e8a6bb0 Strip script and style tags through ::clean() method instead of preg_replace
Huge tags can lead to a failure of preg_replace, thus erasing the whole
fetched content.

Fixes https://github.com/wallabag/wallabag/issues/5847

Signed-off-by: Kevin Decherf <kevin@kdecherf.com>
2022-06-11 00:11:36 +02:00
14 changed files with 542 additions and 682 deletions
-1
View File
@@ -6,6 +6,5 @@
/.php_cs export-ignore
/phpunit.xml.dist export-ignore
/phpstan.neon export-ignore
/rector.php export-ignore
/.github export-ignore
/tests export-ignore
+9 -3
View File
@@ -8,10 +8,13 @@ on:
branches:
- master
env:
SYMFONY_PHPUNIT_VERSION: 7.5
jobs:
coding-standards:
name: "CS Fixer & PHPStan"
runs-on: "ubuntu-22.04"
runs-on: "ubuntu-20.04"
strategy:
matrix:
@@ -20,7 +23,7 @@ jobs:
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
uses: "actions/checkout@v2"
- name: "Install PHP"
uses: "shivammathur/setup-php@v2"
@@ -32,8 +35,11 @@ jobs:
env:
COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Add PHPStan"
run: "composer require phpstan/phpstan phpstan/phpstan-phpunit --dev --no-progress --no-suggest"
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v3"
uses: "ramsey/composer-install@v1"
with:
composer-options: "--optimize-autoloader --prefer-dist"
+71 -18
View File
@@ -14,21 +14,22 @@ env:
jobs:
phpunit:
name: "PHPUnit (PHP ${{ matrix.php }})"
runs-on: "ubuntu-22.04"
runs-on: "ubuntu-20.04"
strategy:
matrix:
php:
- "5.6"
- "7.0"
- "7.1"
- "7.2"
- "7.3"
- "7.4"
- "8.0"
- "8.1"
- "8.2"
- "8.3"
- "8.4"
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
uses: "actions/checkout@v2"
with:
fetch-depth: 2
@@ -37,17 +38,23 @@ jobs:
with:
php-version: "${{ matrix.php }}"
coverage: "none"
tools: composer:v2
tools: composer:v1
extensions: tidy
ini-values: "date.timezone=Europe/Paris"
env:
COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Force PHPUnit version"
if: matrix.php >= '7.2'
run: "echo $SYMFONY_PHPUNIT_VERSION"
env:
SYMFONY_PHPUNIT_VERSION: 7.5
- name: "Remove useless deps"
run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v3"
uses: "ramsey/composer-install@v1"
with:
composer-options: "--optimize-autoloader --prefer-dist"
@@ -59,16 +66,16 @@ jobs:
phpunit-coverage:
name: "PHPUnit coverage (PHP ${{ matrix.php }})"
runs-on: "ubuntu-22.04"
runs-on: "ubuntu-20.04"
strategy:
matrix:
php:
- "8.0"
- "7.4"
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
uses: "actions/checkout@v2"
with:
fetch-depth: 2
@@ -77,7 +84,7 @@ jobs:
with:
php-version: "${{ matrix.php }}"
coverage: "xdebug"
tools: composer:v2
tools: composer:v1
extensions: tidy
ini-values: "date.timezone=Europe/Paris"
env:
@@ -87,7 +94,7 @@ jobs:
run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v3"
uses: "ramsey/composer-install@v1"
with:
composer-options: "--optimize-autoloader --prefer-dist"
@@ -96,9 +103,11 @@ jobs:
- name: "Run PHPUnit (with coverage)"
run: "php vendor/bin/simple-phpunit -v --coverage-clover build/logs/clover.xml"
env:
SYMFONY_PHPUNIT_VERSION: 7.5
- name: "Retrieve Coveralls phar"
run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.7.0/php-coveralls.phar"
run: "wget https://github.com/php-coveralls/php-coveralls/releases/download/v2.4.2/php-coveralls.phar"
- name: "Enable Coveralls phar"
run: "chmod +x php-coveralls.phar"
@@ -110,7 +119,50 @@ jobs:
phpunit-lowest:
name: "PHPUnit lowest deps (PHP ${{ matrix.php }})"
runs-on: "ubuntu-22.04"
runs-on: "ubuntu-20.04"
strategy:
matrix:
php:
- "7.2"
steps:
- name: "Checkout"
uses: "actions/checkout@v2"
with:
fetch-depth: 2
- name: "Install PHP"
uses: "shivammathur/setup-php@v2"
with:
php-version: "${{ matrix.php }}"
coverage: "none"
tools: composer:v1
extensions: tidy
ini-values: "date.timezone=Europe/Paris"
env:
COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Remove useless deps"
run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v1"
with:
composer-options: "--optimize-autoloader --prefer-dist"
dependency-versions: "lowest"
- name: "Setup logs"
run: "mkdir -p build/logs"
- name: "Run PHPUnit"
run: "php vendor/bin/simple-phpunit -v"
env:
SYMFONY_PHPUNIT_VERSION: 7.5
phpunit-composerv2:
name: "PHPUnit with Composer v2 (PHP ${{ matrix.php }})"
runs-on: "ubuntu-20.04"
strategy:
matrix:
@@ -119,7 +171,7 @@ jobs:
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
uses: "actions/checkout@v2"
with:
fetch-depth: 2
@@ -138,13 +190,14 @@ jobs:
run: "composer remove friendsofphp/php-cs-fixer --dev --no-progress --no-update"
- name: "Install dependencies with Composer"
uses: "ramsey/composer-install@v3"
uses: "ramsey/composer-install@v1"
with:
composer-options: "--optimize-autoloader --prefer-dist"
dependency-versions: "lowest"
- name: "Setup logs"
run: "mkdir -p build/logs"
- name: "Run PHPUnit"
run: "php vendor/bin/simple-phpunit -v"
env:
SYMFONY_PHPUNIT_VERSION: 7.5
-2
View File
@@ -2,6 +2,4 @@ vendor/
coverage/
composer.lock
.php_cs.cache
.php-cs-fixer.cache
.phpunit.result.cache
phpstan.neon
-4
View File
@@ -26,10 +26,6 @@ return (new PhpCsFixer\Config())
'strict_comparison' => true,
'strict_param' => true,
'concat_space' => ['spacing' => 'one'],
// Pulled in by @Symfony:risky but we still support PHP 7.4
'modernize_strpos' => false,
// Pulled in by @Symfony, we cannot add property types until we bump PHP to ≥ 7.4
'no_null_property_initialization' => false,
])
->setFinder($finder)
;
+12
View File
@@ -0,0 +1,12 @@
tools:
external_code_coverage: false
build:
nodes:
analysis:
tests:
override:
- php-scrutinizer-run
environment:
php:
version: 7.2
+1 -1
View File
@@ -5,7 +5,7 @@
[![Total Downloads](https://poser.pugx.org/j0k3r/php-readability/downloads)](https://packagist.org/packages/j0k3r/php-readability)
[![License](https://poser.pugx.org/j0k3r/php-readability/license)](https://packagist.org/packages/j0k3r/php-readability)
This is an extract of the Readability class from this [full-text-rss](https://github.com/Dither/full-text-rss) fork. It can be defined as a better version of the original [php-readability](https://bitbucket.org/fivefilters/php-readability).
This is an extract of the Readability class from this [full-text-rss](https://github.com/Dither/full-text-rss) fork. It can be defined as a better version of the original [php-readability](https://bitbucket.org/fivefilters/php-readability/overview).
## Differences
+4 -13
View File
@@ -24,18 +24,15 @@
"role": "Developer (original JS version)"
}],
"require": {
"php": ">=7.4.0",
"php": ">=5.6.0",
"ext-mbstring": "*",
"psr/log": "^1.0.1 || ^2.0 || ^3.0",
"psr/log": "^1.0",
"masterminds/html5": "^2.7"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.0",
"friendsofphp/php-cs-fixer": "^2.14",
"monolog/monolog": "^1.24|^2.1",
"symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0",
"phpstan/phpstan": "^2.0",
"phpstan/phpstan-phpunit": "^2.0",
"rector/rector": "^2.0.0"
"symfony/phpunit-bridge": "^4.4|^5.3"
},
"suggest": {
"ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure."
@@ -45,11 +42,5 @@
},
"autoload-dev": {
"psr-4": { "Tests\\Readability\\": "tests/" }
},
"scripts": {
"fix": "php-cs-fixer fix --verbose --diff",
"phpstan": "phpstan analyze --memory-limit 512M",
"rector": "rector process",
"test": "simple-phpunit -v"
}
}
+1 -1
View File
@@ -6,7 +6,7 @@ parameters:
# https://github.com/phpstan/phpstan/issues/694#issuecomment-350724288
bootstrapFiles:
- vendor/bin/.phpunit/phpunit/vendor/autoload.php
- vendor/bin/.phpunit/phpunit-7.5-0/vendor/autoload.php
includes:
- vendor/phpstan/phpstan-phpunit/extension.neon
+1 -1
View File
@@ -11,7 +11,7 @@
>
<testsuites>
<testsuite name="Readability">
<testsuite name="Readability Test Suite">
<directory>./tests/</directory>
</testsuite>
</testsuites>
-18
View File
@@ -1,18 +0,0 @@
<?php
declare(strict_types=1);
use Rector\Config\RectorConfig;
use Rector\Set\ValueObject\LevelSetList;
return RectorConfig::configure()
->withPaths([
__DIR__ . '/src',
__DIR__ . '/tests',
])
->withBootstrapFiles([
__DIR__ . '/vendor/bin/.phpunit/phpunit/vendor/autoload.php',
__DIR__ . '/vendor/autoload.php',
])
->withSets([LevelSetList::UP_TO_PHP_74])
;
+6 -5
View File
@@ -39,9 +39,9 @@ class JSLikeHTMLElement extends \DOMElement
/**
* Used for setting innerHTML like it's done in JavaScript:.
*
* ```php
* @code
* $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
* ```
* @endcode
*/
public function __set($name, $value)
{
@@ -79,13 +79,14 @@ class JSLikeHTMLElement extends \DOMElement
} else {
// $value is probably ill-formed
$f = new \DOMDocument();
$value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
// Using <htmlfragment> will generate a warning, but so will bad HTML
// (and by this point, bad HTML is what we've got).
// We use it (and suppress the warning) because an HTML fragment will
// be wrapped around <html><body> tags which we don't really want to keep.
// Note: despite the warning, if loadHTML succeeds it will return true.
$result = $f->loadHTML('<meta charset="utf-8"><htmlfragment>' . $value . '</htmlfragment>');
$result = $f->loadHTML('<htmlfragment>' . $value . '</htmlfragment>');
if ($result) {
$import = $f->getElementsByTagName('htmlfragment')->item(0);
@@ -104,9 +105,9 @@ class JSLikeHTMLElement extends \DOMElement
/**
* Used for getting innerHTML like it's done in JavaScript:.
*
* ```php
* @code
* $string = $div->innerHTML;
* ```
* @endcode
*/
public function __get($name)
{
+301 -404
View File
File diff suppressed because it is too large Load Diff
+136 -211
View File
@@ -4,136 +4,129 @@ namespace Tests\Readability;
use Monolog\Handler\TestHandler;
use Monolog\Logger;
use Psr\Log\LoggerInterface;
use Readability\JSLikeHTMLElement;
use Readability\Readability;
class ReadabilityTest extends \PHPUnit\Framework\TestCase
{
/** @var TestHandler */
public $logHandler;
/** @var LoggerInterface */
public $logger;
/**
* @requires extension tidy
*/
public function testConstructDefault(): void
public function testConstructDefault()
{
$readability = $this->getReadability('');
$readability->init();
$this->assertNull($readability->url);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertInstanceOf('DomDocument', $readability->dom);
}
public function testConstructHtml5Parser(): void
public function testConstructHtml5Parser()
{
$readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'html5lib');
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertInstanceOf('DomDocument', $readability->dom);
$this->assertSame('<html/>', $readability->original_html);
}
/**
* @requires extension tidy
*/
public function testConstructSimple(): void
public function testConstructSimple()
{
$readability = $this->getReadability('<html/>', 'http://0.0.0.0');
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertInstanceOf('DomDocument', $readability->dom);
$this->assertSame('<html/>', $readability->original_html);
$this->assertTrue($readability->tidied);
}
public function testConstructDefaultWithoutTidy(): void
public function testConstructDefaultWithoutTidy()
{
$readability = $this->getReadability('', null, 'libxml', false);
$readability->init();
$this->assertNull($readability->url);
$this->assertSame('', $readability->original_html);
$this->assertFalse($readability->tidied);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertInstanceOf('DomDocument', $readability->dom);
}
public function testConstructSimpleWithoutTidy(): void
public function testConstructSimpleWithoutTidy()
{
$readability = $this->getReadability('<html/>', 'http://0.0.0.0', 'libxml', false);
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertInstanceOf('DomDocument', $readability->dom);
$this->assertSame('<html/>', $readability->original_html);
$this->assertFalse($readability->tidied);
}
public function testInitNoContent(): void
public function testInitNoContent()
{
$readability = $this->getReadability('<html/>', 'http://0.0.0.0');
$res = $readability->init();
$this->assertFalse($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('Sorry, Readability was unable to parse this page for content.', $readability->getContent()->getInnerHtml());
}
public function testInitP(): void
public function testInitP()
{
$readability = $this->getReadability(str_repeat('<p>This is the awesome content :)</p>', 7), 'http://0.0.0.0');
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
}
public function testInitDivP(): void
public function testInitDivP()
{
$readability = $this->getReadability('<div>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</div>', 'http://0.0.0.0');
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
}
public function testInitDiv(): void
public function testInitDiv()
{
$readability = $this->getReadability('<div>' . str_repeat('This is the awesome content :)', 7) . '</div>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is the awesome content :)', $readability->getContent()->getInnerHtml());
}
public function testWithFootnotes(): void
public function testWithFootnotes()
{
$readability = $this->getReadability('<div>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '</div>', 'http://0.0.0.0');
$readability->debug = true;
$readability->convertLinksToFootnotes = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
@@ -141,15 +134,16 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringContainsString('readabilityLink-3', $readability->getContent()->getInnerHtml());
}
public function testStandardClean(): void
public function testStandardClean()
{
$readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<a href="#nofollow" rel="nofollow">will NOT be removed</a></div>', 'http://0.0.0.0');
$readability->debug = true;
$readability->lightClean = false;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
@@ -157,224 +151,232 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringNotContainsString('<h2>', $readability->getContent()->getInnerHtml());
}
public function testWithIframe(): void
public function testWithIframe()
{
$readability = $this->getReadability('<div><h2>Title</h2>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe><iframe>http://soundcloud.com/test</iframe></p></div>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<div readability=', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
}
public function testWithArticle(): void
public function testWithArticle()
{
$readability = $this->getReadability('<article><p>' . str_repeat('This is an awesome text with some links, here there are: the awesome', 20) . '</p><p>This is an awesome text with some links, here there are <iframe src="http://youtube.com/test" href="#nofollow" rel="nofollow"></iframe></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('nofollow', $readability->getContent()->getInnerHtml());
}
public function testWithAside(): void
public function testWithAside()
{
$readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<footer><aside>' . str_repeat('<p>This is an awesome text with some links, here there are</p>', 8) . '</aside></footer></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('<aside>', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('<footer readability="9"/>', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('<footer readability="5"/>', $readability->getContent()->getInnerHtml());
}
public function testWithClasses(): void
public function testWithClasses()
{
$readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
}
public function testWithClassesWithoutLightClean(): void
public function testWithClassesWithoutLightClean()
{
$readability = $this->getReadability('<article>' . str_repeat('<p>This is an awesome text with some links, here there are: <a href="http://0.0.0.0/test.html">the awesome</a></p>', 7) . '<div style="display:none">' . str_repeat('<p class="clock">This text should be removed</p>', 10) . '</div></article>', 'http://0.0.0.0');
$readability->debug = true;
$readability->lightClean = false;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('alt="article"', $readability->getContent()->getInnerHtml());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text should be removed', $readability->getContent()->getInnerHtml());
}
public function testWithTd(): void
public function testWithTd()
{
$readability = $this->getReadability('<table><tr>' . str_repeat('<td><p>This is an awesome text with some links, here there are the awesome</td>', 7) . '</tr></table>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
}
public function testWithSameClasses(): void
public function testWithSameClasses()
{
$readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<div class="awesomecontent">This text is also an awesome text and you should know that !</div></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testWithScript(): void
public function testWithScript()
{
$readability = $this->getReadability('<article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p><script>This text is also an awesome text and you should know that !</script></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertEmpty($readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testTitle(): void
public function testTitle()
{
$readability = $this->getReadability('<title>this is my title</title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertSame('this is my title', $readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testTitleWithDash(): void
public function testTitleWithDash()
{
$readability = $this->getReadability('<title> title2 - title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertSame('title2 - title3', $readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testTitleWithDoubleDot(): void
public function testTitleWithDoubleDot()
{
$readability = $this->getReadability('<title> title2 : title3 </title><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertSame('title2 : title3', $readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testTitleTooShortUseH1(): void
public function testTitleTooShortUseH1()
{
$readability = $this->getReadability('<title>too short</title><h1>this is my h1 title !</h1><article class="awesomecontent">' . str_repeat('<p>This is an awesome text with some links, here there are the awesome</p>', 7) . '<p></p></article>', 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertSame('this is my h1 title !', $readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('This is an awesome text with some links, here there are', $readability->getContent()->getInnerHtml());
$this->assertStringNotContainsString('This text is also an awesome text and you should know that', $readability->getContent()->getInnerHtml());
}
public function testAutoClosingIframeNotThrowingException(): void
// dummy function to be used to the next test
public function error2Exception($code, $string, $file, $line, $context)
{
$oldErrorReporting = error_reporting(\E_ALL);
$oldDisplayErrors = ini_set('display_errors', '1');
// dummy function to be used to the next test
set_error_handler(function (int $errno, string $errstr, string $errfile, int $errline) {
throw new \Exception($errstr, $errno);
});
throw new \Exception($string, $code);
}
try {
$data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
public function testAutoClosingIframeNotThrowingException()
{
error_reporting(\E_ALL | \E_STRICT);
ini_set('display_errors', true);
set_error_handler([$this, 'error2Exception'], \E_ALL | \E_STRICT);
<head profile="http://gmpg.org/xfn/11">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
$data = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="ru-RU" prefix="og: http://ogp.me/ns#">
</head>
<body class="single single-post postid-22030 single-format-standard">
<div id="wrapper">
<div id="content">
<div class="post-22030 post type-post status-publish format-standard has-post-thumbnail hentry category-video category-reviews tag-193" id="post-22030">
<h1>3D Touch &#8212; будущее мобильных игр</h1>
<div class="postdate">Автор: <strong>Сергей Пак</strong> | Просмотров: 1363 | Опубликовано: 14 сентября 2015 </div>
<div class="entry">
<p>Компания Apple представила новую технологию 3D Touch, которая является прямым потомком более ранней версии Force Touch &#8212; последняя, напомним, используется сейчас в трекпадах Macbook Pro и Macbook 2015. Теперь управлять устройством стало в разы проще, и Force Touch открывает перед пользователями новые возможности, но при этом 3D Touch &#8212; это про другое. Дело в том, что теперь и на мобильных устройствах интерфейс будет постепенно меняться, кардинальные перемены ждут мобильный гейминг, потому что здесь разработчики действительно могут разгуляться.<span id="more-22030"></span></p>
<p><iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"></iframe></p>
<p>Итак, просто представьте себе, что iPhone 6S &#8212; это, по большому счету, отличная игровая приставка, которую вы носите с собой, а еще она может выдавать невероятной красоты картинку. Но проблема заключается, пожалуй, в том, что управлять персонажем в играх довольно трудно &#8212; он неповоротлив, обладает заторможенной реакцией, а игровой клиент зачастую требует перегруза интерфейса для того, чтобы обеспечить максимально большое количество возможностей. Благодаря трехуровневому нажатию можно избавиться от лишних кнопок и обеспечить более качественный обзор местности, и при этом пользователь будет закрывать пальцами минимальное пространство.</p>
</div>
</div>
<head profile="http://gmpg.org/xfn/11">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
</head>
<body class="single single-post postid-22030 single-format-standard">
<div id="wrapper">
<div id="content">
<div class="post-22030 post type-post status-publish format-standard has-post-thumbnail hentry category-video category-reviews tag-193" id="post-22030">
<h1>3D Touch &#8212; будущее мобильных игр</h1>
<div class="postdate">Автор: <strong>Сергей Пак</strong> | Просмотров: 1363 | Опубликовано: 14 сентября 2015 </div>
<div class="entry">
<p>Компания Apple представила новую технологию 3D Touch, которая является прямым потомком более ранней версии Force Touch &#8212; последняя, напомним, используется сейчас в трекпадах Macbook Pro и Macbook 2015. Теперь управлять устройством стало в разы проще, и Force Touch открывает перед пользователями новые возможности, но при этом 3D Touch &#8212; это про другое. Дело в том, что теперь и на мобильных устройствах интерфейс будет постепенно меняться, кардинальные перемены ждут мобильный гейминг, потому что здесь разработчики действительно могут разгуляться.<span id="more-22030"></span></p>
<p><iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"></iframe></p>
<p>Итак, просто представьте себе, что iPhone 6S &#8212; это, по большому счету, отличная игровая приставка, которую вы носите с собой, а еще она может выдавать невероятной красоты картинку. Но проблема заключается, пожалуй, в том, что управлять персонажем в играх довольно трудно &#8212; он неповоротлив, обладает заторможенной реакцией, а игровой клиент зачастую требует перегруза интерфейса для того, чтобы обеспечить максимально большое количество возможностей. Благодаря трехуровневому нажатию можно избавиться от лишних кнопок и обеспечить более качественный обзор местности, и при этом пользователь будет закрывать пальцами минимальное пространство.</p>
</div>
</div>
</body>
</html>';
</div>
</div>
</body>
</html>';
$readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
$readability = $this->getReadability($data, 'http://iosgames.ru/?p=22030');
$readability->debug = true;
$res = $readability->init();
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
} finally {
restore_error_handler();
if (false !== $oldDisplayErrors) {
ini_set('display_errors', $oldDisplayErrors);
}
error_reporting($oldErrorReporting);
}
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertStringContainsString('<iframe src="https://www.youtube.com/embed/PUep6xNeKjA" width="560" height="315" frameborder="0" allowfullscreen="allowfullscreen"> </iframe>', $readability->getContent()->getInnerHtml());
$this->assertStringContainsString('3D Touch', $readability->getTitle()->getInnerHtml());
}
/**
* This should generate an Exception "DOMElement::setAttribute(): ID post-60 already defined".
*/
public function testAppendIdAlreadyHere(): void
public function testAppendIdAlreadyHere()
{
$data = '<!DOCTYPE html>
<html lang="fr">
@@ -421,15 +423,16 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
</html>';
$readability = $this->getReadability($data, 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getContent());
$this->assertInstanceOf(JSLikeHTMLElement::class, $readability->getTitle());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
}
public function testPostFilters(): void
public function testPostFilters()
{
$readability = $this->getReadability('<div>' . str_repeat('<p>This <strong>is</strong> the awesome content :)</p>', 10) . '</div>', 'http://0.0.0.0');
$readability->addPostFilter('!<strong[^>]*>(.*?)</strong>!is', '');
@@ -440,8 +443,10 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringContainsString('This the awesome content :)', $readability->getContent()->getInnerHtml());
}
public function testPreFilters(): void
public function testPreFilters()
{
$this->markTestSkipped('Won\'t work until loadHtml() is moved in init() instead of __construct()');
$readability = $this->getReadability('<div>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>', 'http://0.0.0.0');
$readability->addPreFilter('!<b[^>]*>(.*?)</b>!is', '');
@@ -451,24 +456,26 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringContainsString('This the awesome and WONDERFUL content :)', $readability->getContent()->getInnerHtml());
}
public function testChildNodeGoneNull(): void
public function testChildNodeGoneNull()
{
// from http://www.ayyaantuu.net/ethiopia-targets-opposition-lawmakers/
$html = (string) file_get_contents('tests/fixtures/childNodeGoesNull.html');
$html = file_get_contents('tests/fixtures/childNodeGoesNull.html');
$readability = $this->getReadability($html, 'http://0.0.0.0');
$readability->debug = true;
$readability->convertLinksToFootnotes = true;
$res = $readability->init();
$this->assertTrue($res);
}
public function testKeepFootnotes(): void
public function testKeepFootnotes()
{
// from https://www.schreibdichte.de/blog/feed-aggregator-und-spaeter-lesen-dienst-im-team
$html = (string) file_get_contents('tests/fixtures/keepFootnotes.html');
$html = file_get_contents('tests/fixtures/keepFootnotes.html');
$readability = $this->getReadability($html, 'http://0.0.0.0');
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
@@ -476,102 +483,20 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringContainsString('<a href="#fnref1:fnfeed_2" rev="footnote"', $readability->getContent()->getInnerHtml());
}
public function testWithWipedBody(): void
public function testWithWipedBody()
{
// from https://www.cs.cmu.edu/~rgs/alice-table.html
$html = (string) file_get_contents('tests/fixtures/wipedBody.html');
$html = file_get_contents('tests/fixtures/wipedBody.html');
$readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', false);
$readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
}
public function dataForVisibleNode(): array
{
return [
'visible node' => [
'<div>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
true,
],
'display=none' => [
'<div style="display:none;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
false,
],
'display=inline' => [
'<div style="display:inline;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
true,
],
'hidden attribute' => [
'<div hidden>' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
false,
],
'missing display' => [
'<div style="color:#ccc;">' . str_repeat('<p>This <b>is</b> the awesome and WONDERFUL content :)</p>', 7) . '</div>',
true,
],
];
}
/**
* @dataProvider dataForVisibleNode
*/
public function testVisibleNode(string $content, bool $shouldBeVisible): void
{
$readability = $this->getReadability($content, 'http://0.0.0.0');
$res = $readability->init();
if ($shouldBeVisible) {
$this->assertStringContainsString('WONDERFUL content', $readability->getContent()->getInnerHtml());
} else {
$this->assertStringNotContainsString('WONDERFUL content', $readability->getContent()->getInnerHtml());
}
}
/**
* @return array<string, array{0: string, 1: string, 2?: bool}>
*/
public function dataForHtmlLang(): array
{
return [
'meta' => [
'<html lang="fr"><head><meta charset="utf-8"></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
'fr',
],
'head' => [
'<html lang="fr"><head><title>Foo</title></head><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
'fr',
],
'headless' => [
'<html lang="fr"><body><article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article></body></html>',
'fr',
// tidy would add <head> tag.
false,
],
'fragment' => [
'<article>' . str_repeat('<p>This is the awesome content :)</p>', 7) . '</article>',
'',
// tidy would add <html>.
false,
],
];
}
/**
* @dataProvider dataForHtmlLang
*/
public function testHtmlLang(string $html, string $lang, bool $useTidy = true): void
{
$readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy);
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang'));
}
private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability
private function getReadability($html, $url = null, $parser = 'libxml', $useTidy = true)
{
$readability = new Readability($html, $url, $parser, $useTidy);