Update year in copyright lines
[Sone.git] / src / main / kotlin / net / pterodactylus / sone / text / SoneTextParser.kt
1 package net.pterodactylus.sone.text
2
3 import freenet.keys.FreenetURI
4 import freenet.support.Base64
5 import net.pterodactylus.sone.data.Sone
6 import net.pterodactylus.sone.data.impl.IdOnlySone
7 import net.pterodactylus.sone.database.PostProvider
8 import net.pterodactylus.sone.database.SoneProvider
9 import net.pterodactylus.sone.text.LinkType.CHK
10 import net.pterodactylus.sone.text.LinkType.FREEMAIL
11 import net.pterodactylus.sone.text.LinkType.HTTP
12 import net.pterodactylus.sone.text.LinkType.HTTPS
13 import net.pterodactylus.sone.text.LinkType.KSK
14 import net.pterodactylus.sone.text.LinkType.POST
15 import net.pterodactylus.sone.text.LinkType.SONE
16 import net.pterodactylus.sone.text.LinkType.SSK
17 import net.pterodactylus.sone.text.LinkType.USK
18 import org.bitpedia.util.Base32
19 import java.net.MalformedURLException
20
21 /**
22  * [Parser] implementation that can recognize Freenet URIs.
23  */
24 class SoneTextParser(private val soneProvider: SoneProvider?, private val postProvider: PostProvider?) {
25
26         fun parse(source: String, context: SoneTextParserContext?) =
27                         source.split("\n")
28                                         .dropWhile { it.trim() == "" }
29                                         .dropLastWhile { it.trim() == "" }
30                                         .mergeMultipleEmptyLines()
31                                         .flatMap { splitLineIntoParts(it, context) }
32                                         .removeEmptyPlainTextParts()
33                                         .mergeAdjacentPlainTextParts()
34
35         private fun splitLineIntoParts(line: String, context: SoneTextParserContext?) =
36                         generateSequence(PlainTextPart("") as Part to line) { remainder ->
37                                 if (remainder.second == "")
38                                         null
39                                 else
40                                         LinkType.values()
41                                                         .mapNotNull { it.findNext(remainder.second) }
42                                                         .minBy { it.position }
43                                                         .let {
44                                                                 when {
45                                                                         it == null -> PlainTextPart(remainder.second) to ""
46                                                                         it.position == 0 -> it.toPart(context) to it.remainder
47                                                                         else -> PlainTextPart(remainder.second.substring(0, it.position)) to (it.link + it.remainder)
48                                                                 }
49                                                         }
50                         }.map { it.first }.toList()
51
52         private fun NextLink.toPart(context: SoneTextParserContext?) = when (linkType) {
53                 KSK, CHK -> try {
54                         FreenetURI(link).let { freenetUri ->
55                                 FreenetLinkPart(
56                                                 link,
57                                                 if (freenetUri.isKSK) {
58                                                         freenetUri.guessableKey
59                                                 } else {
60                                                         freenetUri.metaString ?: freenetUri.docName ?: link.substring(0, 9)
61                                                 },
62                                                 link.split('?').first()
63                                 )
64                         }
65                 } catch (e: MalformedURLException) {
66                         PlainTextPart(link)
67                 }
68                 SSK, USK ->
69                         try {
70                                 FreenetURI(link).let { uri ->
71                                         uri.docName ?: "${uri.keyType}@${uri.routingKey.freenetBase64}"
72                                 }.let { FreenetLinkPart(link, it, trusted = context?.routingKey?.contentEquals(FreenetURI(link).routingKey) == true) }
73                         } catch (e: MalformedURLException) {
74                                 PlainTextPart(link)
75                         }
76                 SONE -> link.substring(7).let { SonePart(soneProvider?.getSone(it) ?: IdOnlySone(it)) }
77                 POST -> postProvider?.getPost(link.substring(7))?.let { PostPart(it) } ?: PlainTextPart(link)
78                 FREEMAIL -> link.indexOf('@').let { atSign ->
79                         link.substring(atSign + 1, link.length - 9).let { freemailId ->
80                                 FreemailPart(link.substring(0, atSign), freemailId, freemailId.decodedId)
81                         }
82                 }
83                 HTTP, HTTPS -> LinkPart(link, link
84                                 .withoutProtocol
85                                 .withoutWwwPrefix
86                                 .withoutUrlParameters
87                                 .withoutMiddlePathComponents
88                                 .withoutTrailingSlash)
89         }
90
91 }
92
93 private fun List<String>.mergeMultipleEmptyLines() = fold(emptyList<String>()) { previous, current ->
94         if (previous.isEmpty()) {
95                 previous + current
96         } else {
97                 if ((previous.last() == "\n") && (current == "")) {
98                         previous
99                 } else {
100                         previous + ("\n" + current)
101                 }
102         }
103 }
104
105 private fun List<Part>.mergeAdjacentPlainTextParts() = fold(emptyList<Part>()) { parts, part ->
106         if ((parts.lastOrNull() is PlainTextPart) && (part is PlainTextPart)) {
107                 parts.dropLast(1) + PlainTextPart(parts.last().text + part.text)
108         } else {
109                 parts + part
110         }
111 }
112
113 private fun List<Part>.removeEmptyPlainTextParts() = filterNot { it == PlainTextPart("") }
114
115 private val String.decodedId: String get() = Base64.encode(Base32.decode(this))
116 private val String.withoutProtocol get() = substring(indexOf("//") + 2)
117 private val String.withoutUrlParameters get() = split('?').first()
118
119 private val String.withoutWwwPrefix
120         get() = split("/")
121                         .replaceFirst { it.split(".").dropWhile { it == "www" }.joinToString(".") }
122                         .joinToString("/")
123
124 private fun <T> List<T>.replaceFirst(replacement: (T) -> T) = mapIndexed { index, element ->
125         if (index == 0) replacement(element) else element
126 }
127
128 private val String.withoutMiddlePathComponents
129         get() = split("/").let {
130                 if (it.size > 2) {
131                         "${it.first()}/…/${it.last()}"
132                 } else {
133                         it.joinToString("/")
134                 }
135         }
136 private val String.withoutTrailingSlash get() = if (endsWith("/")) substring(0, length - 1) else this
137 private val SoneTextParserContext.routingKey: ByteArray? get() = postingSone?.routingKey
138 private val Sone.routingKey: ByteArray get() = Base64.decode(id)
139
140 private enum class LinkType(private val scheme: String, private val freenetLink: Boolean) {
141
142         KSK("KSK@", true),
143         CHK("CHK@", true),
144         SSK("SSK@", true),
145         USK("USK@", true),
146         HTTP("http://", false),
147         HTTPS("https://", false),
148         SONE("sone://", false) {
149                 override fun validateLinkLength(length: Int) = length.takeIf { it == 50 }
150         },
151         POST("post://", false),
152         FREEMAIL("", true) {
153                 override fun findNext(line: String): NextLink? {
154                         val nextFreemailSuffix = line.indexOf(".freemail").takeIf { it >= 54 } ?: return null
155                         if (line[nextFreemailSuffix - 53] != '@') return null
156                         if (!line.substring(nextFreemailSuffix - 52, nextFreemailSuffix).matches(Regex("^[a-z2-7]*\$"))) return null
157                         val firstCharacterIndex = generateSequence(nextFreemailSuffix - 53) {
158                                 it.minus(1).takeIf { (it >= 0) && line[it].validLocalPart }
159                         }.lastOrNull() ?: return null
160                         return NextLink(firstCharacterIndex, this, line.substring(firstCharacterIndex, nextFreemailSuffix + 9), line.substring(nextFreemailSuffix + 9))
161                 }
162
163                 private val Char.validLocalPart get() = (this in ('A'..'Z')) || (this in ('a'..'z')) || (this in ('0'..'9')) || (this == '-') || (this == '_') || (this == '.')
164         };
165
166         open fun findNext(line: String): NextLink? {
167                 val nextLinkPosition = line.indexOf(scheme).takeIf { it != -1 } ?: return null
168                 val endOfLink = line.substring(nextLinkPosition).findEndOfLink().validate() ?: return null
169                 val link = line.substring(nextLinkPosition, nextLinkPosition + endOfLink)
170                 val realNextLinkPosition = if (freenetLink && line.substring(0, nextLinkPosition).endsWith("freenet:")) nextLinkPosition - 8 else nextLinkPosition
171                 return NextLink(realNextLinkPosition, this, link, line.substring(nextLinkPosition + endOfLink))
172         }
173
174         private fun String.findEndOfLink() =
175                         substring(0, whitespace.find(this)?.range?.start ?: length)
176                                         .dropLastWhile(::isPunctuation)
177                                         .upToFirstUnmatchedParen()
178
179         private fun Int.validate() = validateLinkLength(this)
180         protected open fun validateLinkLength(length: Int) = length.takeIf { it > scheme.length }
181
182         private fun String.upToFirstUnmatchedParen() =
183                         foldIndexed(Pair<Int, Int?>(0, null)) { index, (openParens, firstUnmatchedParen), currentChar ->
184                                 when (currentChar) {
185                                         '(' -> (openParens + 1) to firstUnmatchedParen
186                                         ')' -> ((openParens - 1) to (if (openParens == 0) (firstUnmatchedParen ?: index) else firstUnmatchedParen))
187                                         else -> openParens to firstUnmatchedParen
188                                 }
189                         }.second ?: length
190
191 }
192
193 private val punctuationChars = listOf('.', ',', '?', '!')
194 private fun isPunctuation(char: Char) = char in punctuationChars
195
196 private val whitespace = Regex("[\\u000a\u0020\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u200c\u200d\u202f\u205f\u2060\u2800\u3000]")
197
198 private data class NextLink(val position: Int, val linkType: LinkType, val link: String, val remainder: String)
199
200 private val ByteArray.freenetBase64 get() = Base64.encode(this)!!