diff --git a/src/Readability.php b/src/Readability.php
index 1663884..b2b4d09 100644
--- a/src/Readability.php
+++ b/src/Readability.php
@@ -1422,7 +1422,7 @@ class Readability implements LoggerAwareInterface
unset($tidy);
}
- $this->html = '' . (string) $this->html;
+ $this->html = self::ensureMetaCharset((string) $this->html);
if ('html5lib' === $this->parser || 'html5' === $this->parser) {
$this->dom = (new HTML5())->loadHTML($this->html);
@@ -1510,4 +1510,45 @@ class Readability implements LoggerAwareInterface
)
&& !$node->hasAttribute('hidden');
}
+
+ /**
+ * Tries to insert `meta[charset]` tag into the proper place in the passed HTML document.
+ *
+ * `DOMDocument::loadHTML` will parse HTML documents as ISO-8859-1 if there is no `meta[charset]` tag.
+ * This means that UTF-8-encoded HTML fragments such as those coming from JSON-LD `articleBody` field would be parsed with incorrect encoding.
+ * Unfortunately, we cannot just put the tag at the start of the HTML fragment, since that would cause parser to auto-insert a `html` element, losing the attributes of the original `html` tag.
+ *
+ * @param string $html UTF-8 encoded document
+ */
+ private static function ensureMetaCharset(string $html): string
+ {
+ $charsetTag = '';
+
+ // Only look at first 1024 bytes since, according to HTML5 specification,
+ // that’s where elements declaring a character encoding must be located.
+ // https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset
+ $start = substr($html, 0, 1000);
+
+ if (1 === preg_match('/]+charset/i', $start)) {
+ // tag is already present, no need for modification.
+ return $html;
+ }
+
+ if (1 === preg_match('/]*>/i', $start)) {
+ // tag was located, tags go there.
+ $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1);
+
+ return $html;
+ }
+
+ if (1 === preg_match('/]*>/i', $start)) {
+ // tag was located, let’s put it inside and have parser create .
+ $html = preg_replace('/]*>/i', '$0' . $charsetTag, $html, 1);
+
+ return $html;
+ }
+
+ // Fallback – just plop the at the start of the fragment.
+ return $charsetTag . $html;
+ }
}
diff --git a/tests/ReadabilityTest.php b/tests/ReadabilityTest.php
index 8a29629..d32b29e 100644
--- a/tests/ReadabilityTest.php
+++ b/tests/ReadabilityTest.php
@@ -24,7 +24,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$readability->init();
$this->assertNull($readability->url);
- $this->assertInstanceOf('DomDocument', $readability->dom);
+ $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
}
public function testConstructHtml5Parser(): void
@@ -33,7 +33,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
- $this->assertInstanceOf('DomDocument', $readability->dom);
+ $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertSame('', $readability->original_html);
}
@@ -46,7 +46,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
- $this->assertInstanceOf('DomDocument', $readability->dom);
+ $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertSame('', $readability->original_html);
$this->assertTrue($readability->tidied);
}
@@ -60,7 +60,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$this->assertSame('', $readability->original_html);
$this->assertFalse($readability->tidied);
- $this->assertInstanceOf('DomDocument', $readability->dom);
+ $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
}
public function testConstructSimpleWithoutTidy(): void
@@ -69,7 +69,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
$readability->init();
$this->assertSame('http://0.0.0.0', $readability->url);
- $this->assertInstanceOf('DomDocument', $readability->dom);
+ $this->assertInstanceOf(\DOMDocument::class, $readability->dom);
$this->assertSame('', $readability->original_html);
$this->assertFalse($readability->tidied);
}
@@ -115,7 +115,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
public function testInitDiv(): void
{
$readability = $this->getReadability('
' . str_repeat('This is the awesome content :)', 7) . '
', 'http://0.0.0.0');
- $readability->debug = true;
$res = $readability->init();
$this->assertTrue($res);
@@ -129,7 +128,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
public function testWithFootnotes(): void
{
$readability = $this->getReadability('
' . str_repeat('
This is an awesome text with some links, here there are: the awesome
', 7) . '
', 'http://0.0.0.0');
- $readability->debug = true;
$readability->convertLinksToFootnotes = true;
$res = $readability->init();
@@ -146,7 +144,6 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase
public function testStandardClean(): void
{
$readability = $this->getReadability('
Title
' . str_repeat('
This is an awesome text with some links, here there are: the awesome