package net.pterodactylus.sone.core import com.google.common.base.Ticker import com.google.common.cache.Cache import com.google.common.cache.CacheBuilder import freenet.keys.FreenetURI import org.jsoup.Jsoup import org.jsoup.nodes.Document import java.io.ByteArrayInputStream import java.net.URLDecoder import java.nio.charset.Charset import java.text.Normalizer import java.util.concurrent.TimeUnit.MINUTES import javax.activation.MimeType import javax.imageio.ImageIO import javax.inject.Inject /** * [ElementLoader] implementation that uses a simple Guava [com.google.common.cache.Cache]. */ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticker: Ticker): ElementLoader { @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker()) private val loadingLinks: Cache = CacheBuilder.newBuilder().build() private val failureCache: Cache = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build() private val elementCache: Cache = CacheBuilder.newBuilder().build() private val callback = object: FreenetInterface.BackgroundFetchCallback { override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean { return (size > 2097152) || (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html")) } override fun loaded(uri: FreenetURI, mimeTypeText: String, data: ByteArray) { MimeType(mimeTypeText).also { mimeType -> when { mimeType.primaryType == "image" -> { ByteArrayInputStream(data).use { ImageIO.read(it) }?.let { elementCache.get(uri.toString().decode().normalize()) { LinkedElement(uri.toString(), properties = mapOf("type" to "image", "size" to data.size, "sizeHuman" to data.size.human)) } } } mimeType.baseType == "text/html" -> { val document = Jsoup.parse(data.toString(Charset.forName(mimeType.getParameter("charset") ?: "UTF-8"))) elementCache.get(uri.toString().decode().normalize()) { LinkedElement(uri.toString(), properties = mapOf( "type" to "html", "size" to data.size, "sizeHuman" to data.size.human, "title" to document.title().emptyToNull, "description" to (document.metaDescription ?: document.firstNonHeadingParagraph) )) } } } removeLoadingLink(uri) } } override fun failed(uri: FreenetURI) { failureCache.put(uri.toString().decode().normalize(), true) removeLoadingLink(uri) } private fun removeLoadingLink(uri: FreenetURI) { synchronized(loadingLinks) { loadingLinks.invalidate(uri.toString().decode().normalize()) } } } override fun loadElement(link: String): LinkedElement { val normalizedLink = link.decode().normalize() synchronized(loadingLinks) { elementCache.getIfPresent(normalizedLink)?.run { return this } failureCache.getIfPresent(normalizedLink)?.run { return LinkedElement(link, failed = true) } if (loadingLinks.getIfPresent(normalizedLink) == null) { loadingLinks.put(normalizedLink, true) freenetInterface.startFetch(FreenetURI(link), callback) } } return LinkedElement(link, loading = true) } } private fun String.decode() = URLDecoder.decode(this, "UTF-8")!! private fun String.normalize() = Normalizer.normalize(this, Normalizer.Form.NFC)!! private val String?.emptyToNull get() = if (this == "") null else this private val Document.metaDescription: String? get() = head().getElementsByTag("meta") .map { it.attr("name") to it.attr("content") } .firstOrNull { it.first == "description" } ?.second private val Document.firstNonHeadingParagraph: String? get() = body().children() .map { it to it.text() } .filterNot { it.second == "" } .firstOrNull { !it.first.tagName().startsWith("h", ignoreCase = true) } ?.second private val Int.human get() = when (this) { in 0..1023 -> "$this B" in 1024..1048575 -> "${this / 1024} KiB" in 1048576..1073741823 -> "${this / 1048576} MiB" else -> "${this / 1073741824} GiB" }