src/main/kotlin/net/pterodactylus/sone/text/SoneTextParser.kt

   1 package net.pterodactylus.sone.text
   2
   3 import freenet.keys.*
   4 import freenet.support.*
   5 import net.pterodactylus.sone.data.*
   6 import net.pterodactylus.sone.data.impl.*
   7 import net.pterodactylus.sone.database.*
   8 import net.pterodactylus.sone.text.LinkType.*
   9 import net.pterodactylus.sone.text.LinkType.USK
  10 import net.pterodactylus.sone.utils.*
  11 import org.bitpedia.util.*
  12 import java.net.*
  13 import javax.inject.*
  14
  15 /**
  16  * [Parser] implementation that can recognize Freenet URIs.
  17  */
  18 class SoneTextParser @Inject constructor(private val soneProvider: SoneProvider?, private val postProvider: PostProvider?) {
  19
  20         fun parse(source: String, context: SoneTextParserContext?) =
  21                         source.split("\n")
  22                                         .dropWhile { it.trim() == "" }
  23                                         .dropLastWhile { it.trim() == "" }
  24                                         .mergeMultipleEmptyLines()
  25                                         .flatMap { splitLineIntoParts(it, context) }
  26                                         .removeEmptyPlainTextParts()
  27                                         .mergeAdjacentPlainTextParts()
  28
  29         private fun splitLineIntoParts(line: String, context: SoneTextParserContext?) =
  30                         generateSequence(PlainTextPart("") as Part to line) { remainder ->
  31                                 if (remainder.second == "")
  32                                         null
  33                                 else
  34                                         LinkType.values()
  35                                                         .mapNotNull { it.findNext(remainder.second) }
  36                                                         .minBy { it.position }
  37                                                         .let {
  38                                                                 when {
  39                                                                         it == null -> PlainTextPart(remainder.second) to ""
  40                                                                         it.position == 0 -> it.toPart(context) to it.remainder
  41                                                                         else -> PlainTextPart(remainder.second.substring(0, it.position)) to (it.link + it.remainder)
  42                                                                 }
  43                                                         }
  44                         }.map { it.first }.toList()
  45
  46         private val NextLink.linkWithoutBacklink: String
  47                 get() {
  48                         val backlink = link.indexOf("/../")
  49                         val query = link.indexOf("?")
  50                         return if ((backlink > -1) && ((query == -1) || (query > -1) && (backlink < query)))
  51                                 link.substring(0, backlink)
  52                         else
  53                                 link
  54                 }
  55
  56         private fun NextLink.toPart(context: SoneTextParserContext?) = when (linkType) {
  57                 KSK, CHK -> try {
  58                         FreenetURI(linkWithoutBacklink).let { freenetUri ->
  59                                 FreenetLinkPart(
  60                                                 linkWithoutBacklink,
  61                                                 freenetUri.allMetaStrings?.lastOrNull { it != "" } ?: freenetUri.docName ?: linkWithoutBacklink.substring(0, 9),
  62                                                 linkWithoutBacklink.split('?').first()
  63                                 )
  64                         }
  65                 } catch (e: MalformedURLException) {
  66                         PlainTextPart(linkWithoutBacklink)
  67                 }
  68                 SSK, USK ->
  69                         try {
  70                                 FreenetURI(linkWithoutBacklink).let { uri ->
  71                                         uri.allMetaStrings
  72                                                         ?.takeIf { (it.size > 1) || ((it.size == 1) && (it.single() != "")) }
  73                                                         ?.lastOrNull()
  74                                                         ?: uri.docName
  75                                                         ?: "${uri.keyType}@${uri.routingKey.asFreenetBase64}"
  76                                 }.let { FreenetLinkPart(linkWithoutBacklink.removeSuffix("/"), it, trusted = context?.routingKey?.contentEquals(FreenetURI(linkWithoutBacklink).routingKey) == true) }
  77                         } catch (e: MalformedURLException) {
  78                                 PlainTextPart(linkWithoutBacklink)
  79                         }
  80                 SONE -> link.substring(7).let { SonePart(soneProvider?.getSone(it) ?: IdOnlySone(it)) }
  81                 POST -> postProvider?.getPost(link.substring(7))?.let { PostPart(it) } ?: PlainTextPart(link)
  82                 FREEMAIL -> link.indexOf('@').let { atSign ->
  83                         link.substring(atSign + 1, link.length - 9).let { freemailId ->
  84                                 FreemailPart(link.substring(0, atSign), freemailId, freemailId.decodedId)
  85                         }
  86                 }
  87                 HTTP, HTTPS -> LinkPart(link, link
  88                                 .withoutProtocol
  89                                 .withoutWwwPrefix
  90                                 .withoutUrlParameters
  91                                 .withoutMiddlePathComponents
  92                                 .withoutTrailingSlash)
  93         }
  94
  95 }
  96
  97 private fun List<String>.mergeMultipleEmptyLines() = fold(emptyList<String>()) { previous, current ->
  98         if (previous.isEmpty()) {
  99                 previous + current
 100         } else {
 101                 if ((previous.last() == "\n") && (current == "")) {
 102                         previous
 103                 } else {
 104                         previous + ("\n" + current)
 105                 }
 106         }
 107 }
 108
 109 private fun List<Part>.mergeAdjacentPlainTextParts() = fold(emptyList<Part>()) { parts, part ->
 110         if ((parts.lastOrNull() is PlainTextPart) && (part is PlainTextPart)) {
 111                 parts.dropLast(1) + PlainTextPart(parts.last().text + part.text)
 112         } else {
 113                 parts + part
 114         }
 115 }
 116
 117 private fun List<Part>.removeEmptyPlainTextParts() = filterNot { it == PlainTextPart("") }
 118
 119 private val String.decodedId: String get() = Base32.decode(this).asFreenetBase64
 120 private val String.withoutProtocol get() = substring(indexOf("//") + 2)
 121 private val String.withoutUrlParameters get() = split('?').first()
 122
 123 private val String.withoutWwwPrefix
 124         get() = split("/")
 125                         .replaceFirst { it.split(".").dropWhile { it == "www" }.joinToString(".") }
 126                         .joinToString("/")
 127
 128 private fun <T> List<T>.replaceFirst(replacement: (T) -> T) = mapIndexed { index, element ->
 129         if (index == 0) replacement(element) else element
 130 }
 131
 132 private val String.withoutMiddlePathComponents
 133         get() = split("/").let {
 134                 if (it.size > 2) {
 135                         "${it.first()}/…/${it.last()}"
 136                 } else {
 137                         it.joinToString("/")
 138                 }
 139         }
 140 private val String.withoutTrailingSlash get() = if (endsWith("/")) substring(0, length - 1) else this
 141 private val SoneTextParserContext.routingKey: ByteArray? get() = postingSone?.routingKey
 142 private val Sone.routingKey: ByteArray get() = id.fromFreenetBase64
 143
 144 private enum class LinkType(private val scheme: String, private val freenetLink: Boolean) {
 145
 146         KSK("KSK@", true),
 147         CHK("CHK@", true),
 148         SSK("SSK@", true),
 149         USK("USK@", true),
 150         HTTP("http://", false),
 151         HTTPS("https://", false),
 152         SONE("sone://", false) {
 153                 override fun validateLinkLength(length: Int) = length.takeIf { it == 50 }
 154         },
 155         POST("post://", false),
 156         FREEMAIL("", true) {
 157                 override fun findNext(line: String): NextLink? {
 158                         val nextFreemailSuffix = line.indexOf(".freemail").takeIf { it >= 54 } ?: return null
 159                         if (line[nextFreemailSuffix - 53] != '@') return null
 160                         if (!line.substring(nextFreemailSuffix - 52, nextFreemailSuffix).matches(Regex("^[a-z2-7]*\$"))) return null
 161                         val firstCharacterIndex = generateSequence(nextFreemailSuffix - 53) {
 162                                 it.minus(1).takeIf { (it >= 0) && line[it].validLocalPart }
 163                         }.lastOrNull() ?: return null
 164                         return NextLink(firstCharacterIndex, this, line.substring(firstCharacterIndex, nextFreemailSuffix + 9), line.substring(nextFreemailSuffix + 9))
 165                 }
 166
 167                 private val Char.validLocalPart get() = (this in ('A'..'Z')) || (this in ('a'..'z')) || (this in ('0'..'9')) || (this == '-') || (this == '_') || (this == '.')
 168         };
 169
 170         open fun findNext(line: String): NextLink? {
 171                 val nextLinkPosition = line.indexOf(scheme).takeIf { it != -1 } ?: return null
 172                 val endOfLink = line.substring(nextLinkPosition).findEndOfLink().validate() ?: return null
 173                 val link = line.substring(nextLinkPosition, nextLinkPosition + endOfLink)
 174                 val realNextLinkPosition = if (freenetLink && line.substring(0, nextLinkPosition).endsWith("freenet:")) nextLinkPosition - 8 else nextLinkPosition
 175                 return NextLink(realNextLinkPosition, this, link, line.substring(nextLinkPosition + endOfLink))
 176         }
 177
 178         private fun String.findEndOfLink() =
 179                         substring(0, whitespace.find(this)?.range?.start ?: length)
 180                                         .dropLastWhile(::isPunctuation)
 181                                         .upToFirstUnmatchedParen()
 182
 183         private fun Int.validate() = validateLinkLength(this)
 184         protected open fun validateLinkLength(length: Int) = length.takeIf { it > scheme.length }
 185
 186         private fun String.upToFirstUnmatchedParen() =
 187                         foldIndexed(Pair<Int, Int?>(0, null)) { index, (openParens, firstUnmatchedParen), currentChar ->
 188                                 when (currentChar) {
 189                                         '(' -> (openParens + 1) to firstUnmatchedParen
 190                                         ')' -> ((openParens - 1) to (if (openParens == 0) (firstUnmatchedParen ?: index) else firstUnmatchedParen))
 191                                         else -> openParens to firstUnmatchedParen
 192                                 }
 193                         }.second ?: length
 194
 195 }
 196
 197 private val punctuationChars = listOf('.', ',', '?', '!')
 198 private fun isPunctuation(char: Char) = char in punctuationChars
 199
 200 private val whitespace = Regex("[\\u000a\u0020\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u200c\u200d\u202f\u205f\u2060\u2800\u3000]")
 201
 202 private data class NextLink(val position: Int, val linkType: LinkType, val link: String, val remainder: String)