src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt

   1 package net.pterodactylus.sone.core
   2
   3 import com.google.common.base.Ticker
   4 import com.google.common.cache.Cache
   5 import com.google.common.cache.CacheBuilder
   6 import freenet.keys.FreenetURI
   7 import org.jsoup.Jsoup
   8 import org.jsoup.nodes.Document
   9 import java.io.ByteArrayInputStream
  10 import java.net.URLDecoder
  11 import java.nio.charset.Charset
  12 import java.text.Normalizer
  13 import java.util.concurrent.TimeUnit.MINUTES
  14 import java.util.logging.Logger
  15 import javax.activation.MimeType
  16 import javax.imageio.ImageIO
  17 import javax.inject.Inject
  18
  19 /**
  20  * [ElementLoader] implementation that uses a simple Guava [com.google.common.cache.Cache].
  21  */
  22 class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticker: Ticker): ElementLoader {
  23
  24         @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker())
  25
  26         private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build()
  27         private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build()
  28         private val elementCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build()
  29         private val logger = Logger.getLogger(DefaultElementLoader::class.qualifiedName)
  30         private val callback = object: FreenetInterface.BackgroundFetchCallback {
  31                 override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean {
  32                         if (size > 2097152) {
  33                                 logger.fine { "Canceling download of $uri because it’s > 2 MiB." }
  34                                 return true
  35                         }
  36                         if (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html")) {
  37                                 logger.fine { "Canceling download of $uri because of its MIME type, $mimeType." }
  38                                 return true
  39                         }
  40                         return false
  41                 }
  42
  43                 override fun loaded(uri: FreenetURI, mimeTypeText: String, data: ByteArray) {
  44                         MimeType(mimeTypeText).also { mimeType ->
  45                                 when {
  46                                         mimeType.primaryType == "image" -> {
  47                                                 ByteArrayInputStream(data).use {
  48                                                         ImageIO.read(it)
  49                                                 }?.let {
  50                                                         elementCache.get(uri.toString().decode().normalize()) {
  51                                                                 LinkedElement(uri.toString(), properties = mapOf("type" to "image", "size" to data.size, "sizeHuman" to data.size.human))
  52                                                                         .apply {
  53                                                                                 logger.fine("Downloaded image from $link: size=${properties["size"]}.")
  54                                                                         }
  55                                                         }
  56                                                 }
  57                                         }
  58                                         mimeType.baseType == "text/html" -> {
  59                                                 val document = Jsoup.parse(data.toString(Charset.forName(mimeType.getParameter("charset") ?: "UTF-8")))
  60                                                 elementCache.get(uri.toString().decode().normalize()) {
  61                                                         LinkedElement(uri.toString(), properties = mapOf(
  62                                                                         "type" to "html", "size" to data.size, "sizeHuman" to data.size.human,
  63                                                                         "title" to document.title().emptyToNull,
  64                                                                         "description" to (document.metaDescription ?: document.firstNonHeadingParagraph)
  65                                                         )).apply {
  66                                                                 logger.fine { "Extracted information from $link: title=${properties["title"]}, description=${properties["description"]}." }
  67                                                         }
  68                                                 }
  69                                         }
  70                                 }
  71                                 removeLoadingLink(uri)
  72                         }
  73                 }
  74
  75                 override fun failed(uri: FreenetURI) {
  76                         failureCache.put(uri.toString().decode().normalize(), true)
  77                         removeLoadingLink(uri)
  78                         logger.fine { "Download failed for $uri." }
  79                 }
  80
  81                 private fun removeLoadingLink(uri: FreenetURI) {
  82                         synchronized(loadingLinks) {
  83                                 loadingLinks.invalidate(uri.toString().decode().normalize())
  84                         }
  85                 }
  86         }
  87
  88         override fun loadElement(link: String): LinkedElement {
  89                 val normalizedLink = link.decode().normalize()
  90                 synchronized(loadingLinks) {
  91                         elementCache.getIfPresent(normalizedLink)?.run {
  92                                 return this
  93                         }
  94                         failureCache.getIfPresent(normalizedLink)?.run {
  95                                 return LinkedElement(link, failed = true)
  96                         }
  97                         if (loadingLinks.getIfPresent(normalizedLink) == null) {
  98                                 loadingLinks.put(normalizedLink, true)
  99                                 freenetInterface.startFetch(FreenetURI(link), callback)
 100                         }
 101                 }
 102                 return LinkedElement(link, loading = true)
 103         }
 104
 105 }
 106
 107 private fun String.decode() = try { URLDecoder.decode(this, "UTF-8")!! } catch (e: RuntimeException) { freenet.support.Logger.error(DefaultElementLoader::class.java, "Could not decode %s!".format(this), e); throw e }
 108 private fun String.normalize() = Normalizer.normalize(this, Normalizer.Form.NFC)!!
 109 private val String?.emptyToNull get() = if (this == "") null else this
 110
 111 private val Document.metaDescription: String?
 112         get() = head().getElementsByTag("meta")
 113                 .map { it.attr("name") to it.attr("content") }
 114                 .firstOrNull { it.first == "description" }
 115                 ?.second
 116
 117 private val Document.firstNonHeadingParagraph: String?
 118         get() = body().select("div, p")
 119                 .filter { it.textNodes().isNotEmpty() }
 120                 .map { it to it.text() }
 121                 .firstOrNull { it.second != "" }
 122                 ?.second
 123
 124 private val Int.human get() = when (this) {
 125         in 0..1023 -> "$this B"
 126         in 1024..1048575 -> "${this / 1024} KiB"
 127         in 1048576..1073741823 -> "${this / 1048576} MiB"
 128         else -> "${this / 1073741824} GiB"
 129 }