1 package net.pterodactylus.sone.core
3 import com.google.common.base.Ticker
4 import com.google.common.cache.Cache
5 import com.google.common.cache.CacheBuilder
6 import freenet.keys.FreenetURI
8 import org.jsoup.nodes.Document
9 import java.io.ByteArrayInputStream
10 import java.net.URLDecoder
11 import java.nio.charset.Charset
12 import java.text.Normalizer
13 import java.util.concurrent.TimeUnit.MINUTES
14 import java.util.logging.Logger
15 import javax.activation.MimeType
16 import javax.imageio.ImageIO
17 import javax.inject.Inject
20 * [ElementLoader] implementation that uses a simple Guava [com.google.common.cache.Cache].
22 class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticker: Ticker): ElementLoader {
24 @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker())
26 private val loadingLinks: Cache<String, Boolean> = CacheBuilder.newBuilder().build()
27 private val failureCache: Cache<String, Boolean> = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build()
28 private val elementCache: Cache<String, LinkedElement> = CacheBuilder.newBuilder().build()
29 private val logger = Logger.getLogger(DefaultElementLoader::class.qualifiedName)
30 private val callback = object: FreenetInterface.BackgroundFetchCallback {
31 override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean {
33 logger.fine { "Canceling download of $uri because itβs > 2 MiB." }
36 if (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html")) {
37 logger.fine { "Canceling download of $uri because of its MIME type, $mimeType." }
43 override fun loaded(uri: FreenetURI, mimeTypeText: String, data: ByteArray) {
44 MimeType(mimeTypeText).also { mimeType ->
46 mimeType.primaryType == "image" -> {
47 ByteArrayInputStream(data).use {
50 elementCache.get(uri.toString().decode().normalize()) {
51 LinkedElement(uri.toString(), properties = mapOf("type" to "image", "size" to data.size, "sizeHuman" to data.size.human))
53 logger.fine("Downloaded image from $link: size=${properties["size"]}.")
58 mimeType.baseType == "text/html" -> {
59 val document = Jsoup.parse(data.toString(Charset.forName(mimeType.getParameter("charset") ?: "UTF-8")))
60 elementCache.get(uri.toString().decode().normalize()) {
61 LinkedElement(uri.toString(), properties = mapOf(
62 "type" to "html", "size" to data.size, "sizeHuman" to data.size.human,
63 "title" to document.title().emptyToNull,
64 "description" to (document.metaDescription ?: document.firstNonHeadingParagraph)
66 logger.fine { "Extracted information from $link: title=${properties["title"]}, description=${properties["description"]}." }
71 removeLoadingLink(uri)
75 override fun failed(uri: FreenetURI) {
76 failureCache.put(uri.toString().decode().normalize(), true)
77 removeLoadingLink(uri)
78 logger.fine { "Download failed for $uri." }
81 private fun removeLoadingLink(uri: FreenetURI) {
82 synchronized(loadingLinks) {
83 loadingLinks.invalidate(uri.toString().decode().normalize())
88 override fun loadElement(link: String): LinkedElement {
89 val normalizedLink = link.decode().normalize()
90 synchronized(loadingLinks) {
91 elementCache.getIfPresent(normalizedLink)?.run {
94 failureCache.getIfPresent(normalizedLink)?.run {
95 return LinkedElement(link, failed = true)
97 if (loadingLinks.getIfPresent(normalizedLink) == null) {
98 loadingLinks.put(normalizedLink, true)
99 freenetInterface.startFetch(FreenetURI(link), callback)
102 return LinkedElement(link, loading = true)
107 private fun String.decode() = URLDecoder.decode(this, "UTF-8")!!
108 private fun String.normalize() = Normalizer.normalize(this, Normalizer.Form.NFC)!!
109 private val String?.emptyToNull get() = if (this == "") null else this
111 private val Document.metaDescription: String?
112 get() = head().getElementsByTag("meta")
113 .map { it.attr("name") to it.attr("content") }
114 .firstOrNull { it.first == "description" }
117 private val Document.firstNonHeadingParagraph: String?
118 get() = body().select("div, p")
119 .filter { it.textNodes().isNotEmpty() }
120 .map { it to it.text() }
121 .firstOrNull { it.second != "" }
124 private val Int.human get() = when (this) {
125 in 0..1023 -> "$this B"
126 in 1024..1048575 -> "${this / 1024} KiB"
127 in 1048576..1073741823 -> "${this / 1048576} MiB"
128 else -> "${this / 1073741824} GiB"