1 package net.pterodactylus.sone.text
4 import freenet.support.*
5 import net.pterodactylus.sone.data.*
6 import net.pterodactylus.sone.data.impl.*
7 import net.pterodactylus.sone.database.*
8 import net.pterodactylus.sone.text.LinkType.*
9 import net.pterodactylus.sone.text.LinkType.USK
10 import net.pterodactylus.sone.utils.*
11 import org.bitpedia.util.*
16 * [Parser] implementation that can recognize Freenet URIs.
18 class SoneTextParser @Inject constructor(private val soneProvider: SoneProvider?, private val postProvider: PostProvider?) {
20 fun parse(source: String, context: SoneTextParserContext?) =
22 .dropWhile { it.trim() == "" }
23 .dropLastWhile { it.trim() == "" }
24 .mergeMultipleEmptyLines()
25 .flatMap { splitLineIntoParts(it, context) }
26 .removeEmptyPlainTextParts()
27 .mergeAdjacentPlainTextParts()
29 private fun splitLineIntoParts(line: String, context: SoneTextParserContext?) =
30 generateSequence(PlainTextPart("") as Part to line) { remainder ->
31 if (remainder.second == "")
35 .mapNotNull { it.findNext(remainder.second) }
36 .minBy { it.position }
39 it == null -> PlainTextPart(remainder.second) to ""
40 it.position == 0 -> it.toPart(context) to it.remainder
41 else -> PlainTextPart(remainder.second.substring(0, it.position)) to (it.link + it.remainder)
44 }.map { it.first }.toList()
46 private val NextLink.linkWithoutBacklink: String
48 val backlink = link.indexOf("/../")
49 val query = link.indexOf("?")
50 return if ((backlink > -1) && ((query == -1) || (query > -1) && (backlink < query)))
51 link.substring(0, backlink)
56 private fun NextLink.toPart(context: SoneTextParserContext?) = when (linkType) {
58 FreenetURI(linkWithoutBacklink).let { freenetUri ->
61 freenetUri.allMetaStrings?.lastOrNull { it != "" } ?: freenetUri.docName ?: linkWithoutBacklink.substring(0, 9),
62 linkWithoutBacklink.split('?').first()
65 } catch (e: MalformedURLException) {
66 PlainTextPart(linkWithoutBacklink)
70 FreenetURI(linkWithoutBacklink).let { uri ->
72 ?.takeIf { (it.size > 1) || ((it.size == 1) && (it.single() != "")) }
75 ?: "${uri.keyType}@${uri.routingKey.asFreenetBase64}"
76 }.let { FreenetLinkPart(linkWithoutBacklink.removeSuffix("/"), it, trusted = context?.routingKey?.contentEquals(FreenetURI(linkWithoutBacklink).routingKey) == true) }
77 } catch (e: MalformedURLException) {
78 PlainTextPart(linkWithoutBacklink)
80 SONE -> link.substring(7).let { SonePart(soneProvider?.getSone(it) ?: IdOnlySone(it)) }
81 POST -> postProvider?.getPost(link.substring(7))?.let { PostPart(it) } ?: PlainTextPart(link)
82 FREEMAIL -> link.indexOf('@').let { atSign ->
83 link.substring(atSign + 1, link.length - 9).let { freemailId ->
84 FreemailPart(link.substring(0, atSign), freemailId, freemailId.decodedId)
87 HTTP, HTTPS -> LinkPart(link, link
91 .withoutMiddlePathComponents
92 .withoutTrailingSlash)
97 private fun List<String>.mergeMultipleEmptyLines() = fold(emptyList<String>()) { previous, current ->
98 if (previous.isEmpty()) {
101 if ((previous.last() == "\n") && (current == "")) {
104 previous + ("\n" + current)
109 private fun List<Part>.mergeAdjacentPlainTextParts() = fold(emptyList<Part>()) { parts, part ->
110 if ((parts.lastOrNull() is PlainTextPart) && (part is PlainTextPart)) {
111 parts.dropLast(1) + PlainTextPart(parts.last().text + part.text)
117 private fun List<Part>.removeEmptyPlainTextParts() = filterNot { it == PlainTextPart("") }
119 private val String.decodedId: String get() = Base32.decode(this).asFreenetBase64
120 private val String.withoutProtocol get() = substring(indexOf("//") + 2)
121 private val String.withoutUrlParameters get() = split('?').first()
123 private val String.withoutWwwPrefix
125 .replaceFirst { it.split(".").dropWhile { it == "www" }.joinToString(".") }
128 private fun <T> List<T>.replaceFirst(replacement: (T) -> T) = mapIndexed { index, element ->
129 if (index == 0) replacement(element) else element
132 private val String.withoutMiddlePathComponents
133 get() = split("/").let {
135 "${it.first()}/…/${it.last()}"
140 private val String.withoutTrailingSlash get() = if (endsWith("/")) substring(0, length - 1) else this
141 private val SoneTextParserContext.routingKey: ByteArray? get() = postingSone?.routingKey
142 private val Sone.routingKey: ByteArray get() = id.fromFreenetBase64
144 private enum class LinkType(private val scheme: String, private val freenetLink: Boolean) {
150 HTTP("http://", false),
151 HTTPS("https://", false),
152 SONE("sone://", false) {
153 override fun validateLinkLength(length: Int) = length.takeIf { it == 50 }
155 POST("post://", false),
157 override fun findNext(line: String): NextLink? {
158 val nextFreemailSuffix = line.indexOf(".freemail").takeIf { it >= 54 } ?: return null
159 if (line[nextFreemailSuffix - 53] != '@') return null
160 if (!line.substring(nextFreemailSuffix - 52, nextFreemailSuffix).matches(Regex("^[a-z2-7]*\$"))) return null
161 val firstCharacterIndex = generateSequence(nextFreemailSuffix - 53) {
162 it.minus(1).takeIf { (it >= 0) && line[it].validLocalPart }
163 }.lastOrNull() ?: return null
164 return NextLink(firstCharacterIndex, this, line.substring(firstCharacterIndex, nextFreemailSuffix + 9), line.substring(nextFreemailSuffix + 9))
167 private val Char.validLocalPart get() = (this in ('A'..'Z')) || (this in ('a'..'z')) || (this in ('0'..'9')) || (this == '-') || (this == '_') || (this == '.')
170 open fun findNext(line: String): NextLink? {
171 val nextLinkPosition = line.indexOf(scheme).takeIf { it != -1 } ?: return null
172 val endOfLink = line.substring(nextLinkPosition).findEndOfLink().validate() ?: return null
173 val link = line.substring(nextLinkPosition, nextLinkPosition + endOfLink)
174 val realNextLinkPosition = if (freenetLink && line.substring(0, nextLinkPosition).endsWith("freenet:")) nextLinkPosition - 8 else nextLinkPosition
175 return NextLink(realNextLinkPosition, this, link, line.substring(nextLinkPosition + endOfLink))
178 private fun String.findEndOfLink() =
179 substring(0, whitespace.find(this)?.range?.start ?: length)
180 .dropLastWhile(::isPunctuation)
181 .upToFirstUnmatchedParen()
183 private fun Int.validate() = validateLinkLength(this)
184 protected open fun validateLinkLength(length: Int) = length.takeIf { it > scheme.length }
186 private fun String.upToFirstUnmatchedParen() =
187 foldIndexed(Pair<Int, Int?>(0, null)) { index, (openParens, firstUnmatchedParen), currentChar ->
189 '(' -> (openParens + 1) to firstUnmatchedParen
190 ')' -> ((openParens - 1) to (if (openParens == 0) (firstUnmatchedParen ?: index) else firstUnmatchedParen))
191 else -> openParens to firstUnmatchedParen
197 private val punctuationChars = listOf('.', ',', '?', '!')
198 private fun isPunctuation(char: Char) = char in punctuationChars
200 private val whitespace = Regex("[\\u000a\u0020\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u200c\u200d\u202f\u205f\u2060\u2800\u3000]")
202 private data class NextLink(val position: Int, val linkType: LinkType, val link: String, val remainder: String)