1 package net.pterodactylus.sone.text
4 import freenet.support.*
5 import net.pterodactylus.sone.data.*
6 import net.pterodactylus.sone.data.impl.*
7 import net.pterodactylus.sone.database.*
8 import net.pterodactylus.sone.text.LinkType.*
9 import net.pterodactylus.sone.text.LinkType.USK
10 import org.bitpedia.util.*
15 * [Parser] implementation that can recognize Freenet URIs.
17 class SoneTextParser @Inject constructor(private val soneProvider: SoneProvider?, private val postProvider: PostProvider?) {
19 fun parse(source: String, context: SoneTextParserContext?) =
21 .dropWhile { it.trim() == "" }
22 .dropLastWhile { it.trim() == "" }
23 .mergeMultipleEmptyLines()
24 .flatMap { splitLineIntoParts(it, context) }
25 .removeEmptyPlainTextParts()
26 .mergeAdjacentPlainTextParts()
28 private fun splitLineIntoParts(line: String, context: SoneTextParserContext?) =
29 generateSequence(PlainTextPart("") as Part to line) { remainder ->
30 if (remainder.second == "")
34 .mapNotNull { it.findNext(remainder.second) }
35 .minBy { it.position }
38 it == null -> PlainTextPart(remainder.second) to ""
39 it.position == 0 -> it.toPart(context) to it.remainder
40 else -> PlainTextPart(remainder.second.substring(0, it.position)) to (it.link + it.remainder)
43 }.map { it.first }.toList()
45 private val NextLink.linkWithoutBacklink: String
47 val backlink = link.indexOf("/../")
48 val query = link.indexOf("?")
49 return if ((backlink > -1) && ((query == -1) || (query > -1) && (backlink < query)))
50 link.substring(0, backlink)
55 private fun NextLink.toPart(context: SoneTextParserContext?) = when (linkType) {
57 FreenetURI(linkWithoutBacklink).let { freenetUri ->
60 freenetUri.allMetaStrings?.lastOrNull { it != "" } ?: freenetUri.docName ?: linkWithoutBacklink.substring(0, 9),
61 linkWithoutBacklink.split('?').first()
64 } catch (e: MalformedURLException) {
65 PlainTextPart(linkWithoutBacklink)
69 FreenetURI(linkWithoutBacklink).let { uri ->
71 ?.takeIf { (it.size > 1) || ((it.size == 1) && (it.single() != "")) }
74 ?: "${uri.keyType}@${uri.routingKey.freenetBase64}"
75 }.let { FreenetLinkPart(linkWithoutBacklink.removeSuffix("/"), it, trusted = context?.routingKey?.contentEquals(FreenetURI(linkWithoutBacklink).routingKey) == true) }
76 } catch (e: MalformedURLException) {
77 PlainTextPart(linkWithoutBacklink)
79 SONE -> link.substring(7).let { SonePart(soneProvider?.getSone(it) ?: IdOnlySone(it)) }
80 POST -> postProvider?.getPost(link.substring(7))?.let { PostPart(it) } ?: PlainTextPart(link)
81 FREEMAIL -> link.indexOf('@').let { atSign ->
82 link.substring(atSign + 1, link.length - 9).let { freemailId ->
83 FreemailPart(link.substring(0, atSign), freemailId, freemailId.decodedId)
86 HTTP, HTTPS -> LinkPart(link, link
90 .withoutMiddlePathComponents
91 .withoutTrailingSlash)
96 private fun List<String>.mergeMultipleEmptyLines() = fold(emptyList<String>()) { previous, current ->
97 if (previous.isEmpty()) {
100 if ((previous.last() == "\n") && (current == "")) {
103 previous + ("\n" + current)
108 private fun List<Part>.mergeAdjacentPlainTextParts() = fold(emptyList<Part>()) { parts, part ->
109 if ((parts.lastOrNull() is PlainTextPart) && (part is PlainTextPart)) {
110 parts.dropLast(1) + PlainTextPart(parts.last().text + part.text)
116 private fun List<Part>.removeEmptyPlainTextParts() = filterNot { it == PlainTextPart("") }
118 private val String.decodedId: String get() = Base64.encode(Base32.decode(this))
119 private val String.withoutProtocol get() = substring(indexOf("//") + 2)
120 private val String.withoutUrlParameters get() = split('?').first()
122 private val String.withoutWwwPrefix
124 .replaceFirst { it.split(".").dropWhile { it == "www" }.joinToString(".") }
127 private fun <T> List<T>.replaceFirst(replacement: (T) -> T) = mapIndexed { index, element ->
128 if (index == 0) replacement(element) else element
131 private val String.withoutMiddlePathComponents
132 get() = split("/").let {
134 "${it.first()}/…/${it.last()}"
139 private val String.withoutTrailingSlash get() = if (endsWith("/")) substring(0, length - 1) else this
140 private val SoneTextParserContext.routingKey: ByteArray? get() = postingSone?.routingKey
141 private val Sone.routingKey: ByteArray get() = Base64.decode(id)
143 private enum class LinkType(private val scheme: String, private val freenetLink: Boolean) {
149 HTTP("http://", false),
150 HTTPS("https://", false),
151 SONE("sone://", false) {
152 override fun validateLinkLength(length: Int) = length.takeIf { it == 50 }
154 POST("post://", false),
156 override fun findNext(line: String): NextLink? {
157 val nextFreemailSuffix = line.indexOf(".freemail").takeIf { it >= 54 } ?: return null
158 if (line[nextFreemailSuffix - 53] != '@') return null
159 if (!line.substring(nextFreemailSuffix - 52, nextFreemailSuffix).matches(Regex("^[a-z2-7]*\$"))) return null
160 val firstCharacterIndex = generateSequence(nextFreemailSuffix - 53) {
161 it.minus(1).takeIf { (it >= 0) && line[it].validLocalPart }
162 }.lastOrNull() ?: return null
163 return NextLink(firstCharacterIndex, this, line.substring(firstCharacterIndex, nextFreemailSuffix + 9), line.substring(nextFreemailSuffix + 9))
166 private val Char.validLocalPart get() = (this in ('A'..'Z')) || (this in ('a'..'z')) || (this in ('0'..'9')) || (this == '-') || (this == '_') || (this == '.')
169 open fun findNext(line: String): NextLink? {
170 val nextLinkPosition = line.indexOf(scheme).takeIf { it != -1 } ?: return null
171 val endOfLink = line.substring(nextLinkPosition).findEndOfLink().validate() ?: return null
172 val link = line.substring(nextLinkPosition, nextLinkPosition + endOfLink)
173 val realNextLinkPosition = if (freenetLink && line.substring(0, nextLinkPosition).endsWith("freenet:")) nextLinkPosition - 8 else nextLinkPosition
174 return NextLink(realNextLinkPosition, this, link, line.substring(nextLinkPosition + endOfLink))
177 private fun String.findEndOfLink() =
178 substring(0, whitespace.find(this)?.range?.start ?: length)
179 .dropLastWhile(::isPunctuation)
180 .upToFirstUnmatchedParen()
182 private fun Int.validate() = validateLinkLength(this)
183 protected open fun validateLinkLength(length: Int) = length.takeIf { it > scheme.length }
185 private fun String.upToFirstUnmatchedParen() =
186 foldIndexed(Pair<Int, Int?>(0, null)) { index, (openParens, firstUnmatchedParen), currentChar ->
188 '(' -> (openParens + 1) to firstUnmatchedParen
189 ')' -> ((openParens - 1) to (if (openParens == 0) (firstUnmatchedParen ?: index) else firstUnmatchedParen))
190 else -> openParens to firstUnmatchedParen
196 private val punctuationChars = listOf('.', ',', '?', '!')
197 private fun isPunctuation(char: Char) = char in punctuationChars
199 private val whitespace = Regex("[\\u000a\u0020\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u200c\u200d\u202f\u205f\u2060\u2800\u3000]")
201 private data class NextLink(val position: Int, val linkType: LinkType, val link: String, val remainder: String)
203 private val ByteArray.freenetBase64 get() = Base64.encode(this)!!