projects / Sone.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 17a6598)
🚸 Improve text extraction even further
authorDavid ‘Bombe’ Roden <bombe@pterodactylus.net>
Fri, 2 Sep 2022 15:27:11 +0000 (17:27 +0200)
committerDavid ‘Bombe’ Roden <bombe@pterodactylus.net>
Fri, 2 Sep 2022 15:59:29 +0000 (17:59 +0200)
This actually revamps the way the first paragraph is extracted from freesites and could cause a lot more descriptions to show up in Sone — which was the goal!

Previously I tried to locate all top-level nodes (under <body>) that themselves had text nodes below them and whose name did not start with an “h” (to exclude the header tags) but it turns out this can be easily defeated by wrapping all of the site in e.g. a <center> tag. And I’m sure that a <div> tag would do exactly the same…

So now I use a CSS selector query to get all <p> and <div> nodes, get those with text nodes below them and then get their text (which flattens them for me and removes embedded tags like <a> or <span>).

src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt patch | blob | history
src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt patch | blob | history
src/test/resources/net/pterodactylus/sone/core/element-loader5.html [new file with mode: 0644] patch | blob

diff --git a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
index 88cb1f5..69872c5 100644 (file)
--- a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
+++ b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
@@ -99,10 +99,10 @@ private val Document.metaDescription: String?
                ?.second
 
 private val Document.firstNonHeadingParagraph: String?
-       get() = body().children()
+       get() = body().select("div, p")
+               .filter { it.textNodes().isNotEmpty() }
                .map { it to it.text() }
-               .filterNot { it.second == "" }
-               .firstOrNull { !it.first.tagName().startsWith("h", ignoreCase = true) }
+               .firstOrNull { it.second != "" }
                ?.second
 
 private val Int.human get() = when (this) {
diff --git a/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt b/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt
index 8a3d40f..64456bc 100644 (file)
--- a/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt
+++ b/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt
@@ -151,6 +151,18 @@ class DefaultElementLoaderTest {
        }
 
        @Test
+       fun `element loader can extract first paragraph from real-world example`() {
+               runWithCallback(textKey) { elementLoader, _, callback, _ ->
+                       callback.loaded(FreenetURI(textKey), "text/html; charset=UTF-8", read("element-loader5.html"))
+                       val linkedElement = elementLoader.loadElement(textKey)
+                       assertThat(linkedElement, isLinkedElement(equalTo(textKey), allOf(
+                               hasEntry("type", "html"), hasEntry("title", "Some Nice Page Title"),
+                               hasEntry("description", "This is the first paragraph of the very nice freesite.")
+                       )))
+               }
+       }
+
+       @Test
        fun `image is not loaded again after it failed`() {
                runWithCallback(IMAGE_ID) { elementLoader, _, callback, _ ->
                        elementLoader.loadElement(IMAGE_ID)
diff --git a/src/test/resources/net/pterodactylus/sone/core/element-loader5.html b/src/test/resources/net/pterodactylus/sone/core/element-loader5.html
new file mode 100644 (file)
index 0000000..94fe1b0
--- /dev/null
+++ b/src/test/resources/net/pterodactylus/sone/core/element-loader5.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+<head>
+       <title>Some Nice Page Title</title>
+</head>
+<body>
+<center>
+<h1>First Paragraph</h1>
+<p>This is the <a href="#foo">first paragraph</a> of the very nice freesite.</p>
+</center>
+</body>
+</html>
Sone – The Social Network plugin for Freenet