🚸 Improve text extraction even further

[Sone.git] / src / main / kotlin / net / pterodactylus / sone / core / DefaultElementLoader.kt

diff --git a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
index d01aeb4..69872c5 100644 (file)
--- a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
+++ b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
@@ -6,7 +6,6 @@ import com.google.common.cache.CacheBuilder
 import freenet.keys.FreenetURI
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
-import org.jsoup.nodes.TextNode
 import java.io.ByteArrayInputStream
 import java.net.URLDecoder
 import java.nio.charset.Charset
@@ -23,9 +22,9 @@ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticke
 
        @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker())
 
-       private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build<String, Boolean>()
-       private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build<String, Boolean>()
-       private val elementCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build<String, LinkedElement>()
+       private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build()
+       private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build()
+       private val elementCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build()
        private val callback = object: FreenetInterface.BackgroundFetchCallback {
                override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean {
                        return (size > 2097152) || (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html"))
@@ -58,29 +57,6 @@ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticke
                        }
                }
 
-               private val String?.emptyToNull get() = if (this == "") null else this
-
-               private val Document.metaDescription: String?
-                       get() = head().getElementsByTag("meta")
-                                       .map { it.attr("name") to it.attr("content") }
-                                       .firstOrNull { it.first == "description" }
-                                       ?.second
-
-               private val Document.firstNonHeadingParagraph: String?
-                       get() = body().children()
-                                       .filter { it.children().all { it is TextNode } }
-                                       .map { it to it.text() }
-                                       .filterNot { it.second == "" }
-                                       .firstOrNull { !it.first.tagName().startsWith("h", ignoreCase = true) }
-                                       ?.second
-
-               private val Int.human get() = when (this) {
-                       in 0..1023 -> "$this B"
-                       in 1024..1048575 -> "${this / 1024} KiB"
-                       in 1048576..1073741823 -> "${this / 1048576} MiB"
-                       else -> "${this / 1073741824} GiB"
-               }
-
                override fun failed(uri: FreenetURI) {
                        failureCache.put(uri.toString().decode().normalize(), true)
                        removeLoadingLink(uri)
@@ -110,7 +86,28 @@ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticke
                return LinkedElement(link, loading = true)
        }
 
-       private fun String.decode() = URLDecoder.decode(this, "UTF-8")!!
-       private fun String.normalize() = Normalizer.normalize(this, Normalizer.Form.NFC)!!
+}
+
+private fun String.decode() = URLDecoder.decode(this, "UTF-8")!!
+private fun String.normalize() = Normalizer.normalize(this, Normalizer.Form.NFC)!!
+private val String?.emptyToNull get() = if (this == "") null else this
+
+private val Document.metaDescription: String?
+       get() = head().getElementsByTag("meta")
+               .map { it.attr("name") to it.attr("content") }
+               .firstOrNull { it.first == "description" }
+               ?.second
+
+private val Document.firstNonHeadingParagraph: String?
+       get() = body().select("div, p")
+               .filter { it.textNodes().isNotEmpty() }
+               .map { it to it.text() }
+               .firstOrNull { it.second != "" }
+               ?.second
 
+private val Int.human get() = when (this) {
+       in 0..1023 -> "$this B"
+       in 1024..1048575 -> "${this / 1024} KiB"
+       in 1048576..1073741823 -> "${this / 1048576} MiB"
+       else -> "${this / 1073741824} GiB"
 }