infong
/
php-readability
mirror of https://github.com/j0k3r/php-readability
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix character decoding regression when `title` precedes `meta[charset]`

Browse Source 
Because of PHP 8.2 deprecation, in f14428e4c0, we stopped converting non-ASCII characters to HTML entities. Instead, we started to explicitly insert `meta[charset]` tag at the start of the document.

Later, we discovered that was breaking `html[lang]` so, in efbbc86df9, we made the insertion smarter. One of the improvements was that it would not insert the `meta[charset]` tag when it was already present.

That, however, broke websites that had `title` tag before `meta[charset]`. On those, libxml2 would decode the `title` contents as ISO-8859-1.

We could improve the logic (e.g. check that there is not text content before `meta[charset]`) or insert the tag unconditionally but it will probably be simplest to just go back to converting the non-ASCII characters to entities, just using non-deprecated function variant.
pull/107/head
Jan Tojnar 10 months ago committed by Jeremy Benoist
parent 7f304d03aa
commit 40219d4595
No known key found for this signature in database
GPG Key ID: 7168D5DD29F38552
2 changed files with 21 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 38
      src/Readability.php
  2. 14
      tests/ReadabilityTest.php

38
src/Readability.php
Unescape View File

@ -1435,7 +1435,7 @@ class Readability implements LoggerAwareInterface
unset($tidy); unset($tidy);
} }
$this->html = self::ensureMetaCharset((string) $this->html); $this->html = self::entitizeNonAscii((string) $this->html);
if ('html5lib' === $this->parser || 'html5' === $this->parser) { if ('html5lib' === $this->parser || 'html5' === $this->parser) {
$this->dom = (new HTML5())->loadHTML($this->html); $this->dom = (new HTML5())->loadHTML($this->html);
@ -1455,43 +1455,19 @@ class Readability implements LoggerAwareInterface
} }
/** /**
* Tries to insert `meta[charset]` tag into the proper place in the passed HTML document. * Converts non-ASCII UTF-8 characters to numeric HTML entities.
* *
* `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag. * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
* This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding. * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
* Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag.
* *
* @param string $html UTF-8 encoded document * @param string $html UTF-8 encoded document
*/ */
private static function ensureMetaCharset($html) private static function entitizeNonAscii($html)
{ {
$charsetTag = '<meta charset="utf-8">'; $convmap = [
0x80, 0x1FFFFF, 0, 0x10FFFF,
// Only look at first 1024 bytes since, according to HTML5 specification, ];
// that’s where <meta> elements declaring a character encoding must be located.
// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset
$start = substr($html, 0, 1000);
if (1 === preg_match('/<meta[^>]+charset/i', $start)) {
// <meta> tag is already present, no need for modification.
return $html;
}
if (1 === preg_match('/<head[^>]*>/i', $start)) {
// <head> tag was located, <meta> tags go there.
$html = preg_replace('/<head[^>]*>/i', '$0' . $charsetTag, $html, 1);
return $html;
}
if (1 === preg_match('/<html[^>]*>/i', $start)) {
// <html> tag was located, let’s put it inside and have parser create <head>.
$html = preg_replace('/<html[^>]*>/i', '$0' . $charsetTag, $html, 1);
return $html;
}
// Fallback – just plop the <meta> at the start of the fragment. return mb_encode_numericentity($html, $convmap, 'utf8', true);
return $charsetTag . $html;
} }
} }

14
tests/ReadabilityTest.php
Unescape View File

@ -486,6 +486,20 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml()); $this->assertStringContainsString('<a href="alice-I.html">Down the Rabbit-Hole</a>', $readability->getContent()->getInnerHtml());
} }
// https://github.com/wallabag/wallabag/issues/8158
public function testCharsetAfterTitle()
{
$readability = $this->getReadability('<!DOCTYPE html><html lang="et"><head><title>Tõde ja õigus I</title> <meta charset="utf-8"></head><body><p>See oli läinud aastasaja kolmanda veerandi lõpul. Päike lähenes silmapiirile, seistes sedavõrd madalas, et enam ei ulatunud valgustama ei mäkke ronivat hobust, kes puutelgedega vankrit vedas, ei vankril istuvat noort naist ega ka ligi kolmekümnelist meest, kes kõndis vankri kõrval.</p></body></html>', 'https://et.wikisource.org/wiki/T%C3%B5de_ja_%C3%B5igus_I/I');
$readability->convertLinksToFootnotes = true;
$res = $readability->init();
$this->assertTrue($res);
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getContent());
$this->assertInstanceOf('Readability\JSLikeHTMLElement', $readability->getTitle());
$this->assertSame('Tõde ja õigus I', $readability->getTitle()->getInnerHtml());
$this->assertStringContainsString('Päike lähenes', $readability->getContent()->getInnerHtml());
}
/** /**
* @return array<string, array{0: string, 1: string, 2?: bool}> * @return array<string, array{0: string, 1: string, 2?: bool}>
*/ */

Write Preview
Loading…
Cancel
Save