🔀 Merge branch 'release/v82'

[Sone.git] / src / main / kotlin / net / pterodactylus / sone / core / DefaultElementLoader.kt

diff --git a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
index 5758c47..409d18c 100644 (file)
--- a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
+++ b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt
@@ -4,38 +4,81 @@ import com.google.common.base.Ticker
 import com.google.common.cache.Cache
 import com.google.common.cache.CacheBuilder
 import freenet.keys.FreenetURI
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.TextNode
 import java.io.ByteArrayInputStream
 import java.net.URLDecoder
+import java.nio.charset.Charset
 import java.text.Normalizer
 import java.util.concurrent.TimeUnit.MINUTES
+import javax.activation.MimeType
 import javax.imageio.ImageIO
 import javax.inject.Inject
 
 /**
  * [ElementLoader] implementation that uses a simple Guava [com.google.common.cache.Cache].
  */
-class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticker: Ticker) : ElementLoader {
+class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticker: Ticker): ElementLoader {
 
-       @Inject constructor(freenetInterface: FreenetInterface) : this(freenetInterface, Ticker.systemTicker())
+       @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker())
 
-       private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build<String, Boolean>()
-       private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build<String, Boolean>()
-       private val imageCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build<String, LinkedElement>()
-       private val callback = object : FreenetInterface.BackgroundFetchCallback {
+       private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build()
+       private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build()
+       private val elementCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build()
+       private val callback = object: FreenetInterface.BackgroundFetchCallback {
                override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean {
-                       return !mimeType.startsWith("image/") || (size > 2097152)
+                       return (size > 2097152) || (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html"))
                }
 
-               override fun loaded(uri: FreenetURI, mimeType: String, data: ByteArray) {
-                       if (!mimeType.startsWith("image/")) {
-                               return
+               override fun loaded(uri: FreenetURI, mimeTypeText: String, data: ByteArray) {
+                       MimeType(mimeTypeText).also { mimeType ->
+                               when {
+                                       mimeType.primaryType == "image" -> {
+                                               ByteArrayInputStream(data).use {
+                                                       ImageIO.read(it)
+                                               }?.let {
+                                                       elementCache.get(uri.toString().decode().normalize()) {
+                                                               LinkedElement(uri.toString(), properties = mapOf("type" to "image", "size" to data.size, "sizeHuman" to data.size.human))
+                                                       }
+                                               }
+                                       }
+                                       mimeType.baseType == "text/html" -> {
+                                               val document = Jsoup.parse(data.toString(Charset.forName(mimeType.getParameter("charset") ?: "UTF-8")))
+                                               elementCache.get(uri.toString().decode().normalize()) {
+                                                       LinkedElement(uri.toString(), properties = mapOf(
+                                                                       "type" to "html", "size" to data.size, "sizeHuman" to data.size.human,
+                                                                       "title" to document.title().emptyToNull,
+                                                                       "description" to (document.metaDescription ?: document.firstNonHeadingParagraph)
+                                                       ))
+                                               }
+                                       }
+                               }
+                               removeLoadingLink(uri)
                        }
-                       ByteArrayInputStream(data).use {
-                               ImageIO.read(it)
-                       }?.let {
-                               imageCache.get(uri.toString().decode().normalize()) { LinkedElement(uri.toString()) }
-                       }
-                       removeLoadingLink(uri)
+               }
+
+               private val String?.emptyToNull get() = if (this == "") null else this
+
+               private val Document.metaDescription: String?
+                       get() = head().getElementsByTag("meta")
+                                       .map { it.attr("name") to it.attr("content") }
+                                       .firstOrNull { it.first == "description" }
+                                       ?.second
+
+               private val Document.firstNonHeadingParagraph: String?
+                       get() = body().children()
+                                       .filter { it.children().all { it is TextNode } }
+                                       .map { it to it.text() }
+                                       .filterNot { it.second == "" }
+                                       .firstOrNull { !it.first.tagName().startsWith("h", ignoreCase = true) }
+                                       ?.second
+
+               private val Int.human get() = when (this) {
+                       in 0..1023 -> "$this B"
+                       in 1024..1048575 -> "${this / 1024} KiB"
+                       in 1048576..1073741823 -> "${this / 1048576} MiB"
+                       else -> "${this / 1073741824} GiB"
                }
 
                override fun failed(uri: FreenetURI) {
@@ -53,7 +96,7 @@ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticke
        override fun loadElement(link: String): LinkedElement {
                val normalizedLink = link.decode().normalize()
                synchronized(loadingLinks) {
-                       imageCache.getIfPresent(normalizedLink)?.run {
+                       elementCache.getIfPresent(normalizedLink)?.run {
                                return this
                        }
                        failureCache.getIfPresent(normalizedLink)?.run {