diff --git a/src/Readability.php b/src/Readability.php index 290cc28..7a7d570 100644 --- a/src/Readability.php +++ b/src/Readability.php @@ -1419,7 +1419,7 @@ class Readability implements LoggerAwareInterface unset($tidy); } - $this->html = '' . (string) $this->html; + $this->html = self::ensureMetaCharset((string) $this->html); if ('html5lib' === $this->parser || 'html5' === $this->parser) { $this->dom = (new HTML5())->loadHTML($this->html); @@ -1507,4 +1507,45 @@ class Readability implements LoggerAwareInterface ) && !$node->hasAttribute('hidden'); } + + /** + * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document. + * + * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag. + * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding. + * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag. + * + * @param string $html UTF-8 encoded document + */ + private static function ensureMetaCharset(string $html): string + { + $charsetTag = ''; + + // Only look at first 1024 bytes since, according to HTML5 specification, + // that’s where elements declaring a character encoding must be located. + // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset + $start = substr($html, 0, 1000); + + if (1 === preg_match('/]+charset/i', $start)) { + // tag is already present, no need for modification. + return $html; + } + + if (1 === preg_match('/]*>/i', $start)) { + // tag was located, tags go there. + $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1); + + return $html; + } + + if (1 === preg_match('/]*>/i', $start)) { + // tag was located, let’s put it inside and have parser create . + $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1); + + return $html; + } + + // Fallback – just plop the at the start of the fragment. + return $charsetTag . $html; + } } diff --git a/tests/ReadabilityTest.php b/tests/ReadabilityTest.php index f106b5a..d32b29e 100644 --- a/tests/ReadabilityTest.php +++ b/tests/ReadabilityTest.php @@ -529,6 +529,48 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase } } + /** + * @return array + */ + public function dataForHtmlLang(): array + { + return [ + 'meta' => [ + '
' . str_repeat('

This is the awesome content :)

', 7) . '
', + 'fr', + ], + 'head' => [ + 'Foo
' . str_repeat('

This is the awesome content :)

', 7) . '
', + 'fr', + ], + 'headless' => [ + '
' . str_repeat('

This is the awesome content :)

', 7) . '
', + 'fr', + // tidy would add tag. + false, + ], + 'fragment' => [ + '
' . str_repeat('

This is the awesome content :)

', 7) . '
', + '', + // tidy would add . + false, + ], + ]; + } + + /** + * @dataProvider dataForHtmlLang + */ + public function testHtmlLang(string $html, string $lang, bool $useTidy = true): void + { + $readability = $this->getReadability($html, 'http://0.0.0.0', 'libxml', $useTidy); + $res = $readability->init(); + + $this->assertTrue($res); + $this->assertInstanceOf(\DOMDocument::class, $readability->dom); + $this->assertSame($lang, $readability->dom->documentElement->getAttribute('lang')); + } + private function getReadability(string $html, ?string $url = null, string $parser = 'libxml', bool $useTidy = true): Readability { $readability = new Readability($html, $url, $parser, $useTidy);