From: David ‘Bombe’ Roden Date: Wed, 18 Sep 2019 19:23:32 +0000 (+0200) Subject: 🔀 Merge other “next” branch into this “next” X-Git-Tag: v81^2~127 X-Git-Url: https://git.pterodactylus.net/?p=Sone.git;a=commitdiff_plain;h=b1f3c46e6c76d216b70733c334642f87187676a9;hp=2c0be47db6b6a236d918d86dc801c738b61f7208 🔀 Merge other “next” branch into this “next” --- diff --git a/build.gradle b/build.gradle index 9505ab6..4fd4ada 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ group = 'net.pterodactylus' version = '80' buildscript { - ext.kotlinVersion = '1.3.41' + ext.kotlinVersion = '1.3.50' repositories { mavenCentral() } @@ -54,10 +54,11 @@ dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.2' compile group: 'org.jsoup', name: 'jsoup', version: '1.10.2' compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '4.1.0' + compile group: 'javax.activation', name: 'javax.activation-api', version: '1.2.0' testCompile group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit' testCompile group: 'junit', name: 'junit', version: '4.11' - testCompile group: 'org.mockito', name: 'mockito-core', version: '2.10.0' + testCompile group: 'org.mockito', name: 'mockito-core', version: '2.28.2' testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' } @@ -89,7 +90,7 @@ javadoc { apply plugin: 'jacoco' jacoco { - toolVersion = '0.7.9' + toolVersion = '0.8.4' } jacocoTestReport.dependsOn test diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index c19a936..1cdded7 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.1-all.zip diff --git a/src/main/java/net/pterodactylus/sone/database/memory/MemoryDatabase.kt b/src/main/java/net/pterodactylus/sone/database/memory/MemoryDatabase.kt index a9459e8..857560e 100644 --- a/src/main/java/net/pterodactylus/sone/database/memory/MemoryDatabase.kt +++ b/src/main/java/net/pterodactylus/sone/database/memory/MemoryDatabase.kt @@ -41,7 +41,7 @@ import net.pterodactylus.sone.database.ImageBuilder import net.pterodactylus.sone.database.PostBuilder import net.pterodactylus.sone.database.PostDatabase import net.pterodactylus.sone.database.PostReplyBuilder -import net.pterodactylus.sone.utils.unit +import net.pterodactylus.sone.utils.* import net.pterodactylus.util.config.Configuration import net.pterodactylus.util.config.ConfigurationException import java.util.concurrent.locks.ReentrantReadWriteLock @@ -71,6 +71,8 @@ class MemoryDatabase @Inject constructor(private val configuration: Configuratio private val memoryBookmarkDatabase = MemoryBookmarkDatabase(this, configurationLoader) private val memoryFriendDatabase = MemoryFriendDatabase(configurationLoader) private val saveRateLimiter: RateLimiter = RateLimiter.create(1.0) + private val saveKnownPostsRateLimiter: RateLimiter = RateLimiter.create(1.0) + private val saveKnownPostRepliesRateLimiter: RateLimiter = RateLimiter.create(1.0) override val soneLoader get() = this::getSone @@ -314,15 +316,17 @@ class MemoryDatabase @Inject constructor(private val configuration: Configuratio } private fun saveKnownPosts() = - try { - readLock.withLock { - knownPosts.forEachIndexed { index, knownPostId -> - configuration.getStringValue("KnownPosts/$index/ID").value = knownPostId + saveKnownPostsRateLimiter.tryAcquire().ifTrue { + try { + readLock.withLock { + knownPosts.forEachIndexed { index, knownPostId -> + configuration.getStringValue("KnownPosts/$index/ID").value = knownPostId + } + configuration.getStringValue("KnownPosts/${knownPosts.size}/ID").value = null } - configuration.getStringValue("KnownPosts/${knownPosts.size}/ID").value = null + } catch (ce1: ConfigurationException) { + throw DatabaseException("Could not save database.", ce1) } - } catch (ce1: ConfigurationException) { - throw DatabaseException("Could not save database.", ce1) } private fun loadKnownPostReplies(): Unit = @@ -334,15 +338,17 @@ class MemoryDatabase @Inject constructor(private val configuration: Configuratio } private fun saveKnownPostReplies() = - try { - readLock.withLock { - knownPostReplies.forEachIndexed { index, knownPostReply -> - configuration.getStringValue("KnownReplies/$index/ID").value = knownPostReply + saveKnownPostRepliesRateLimiter.tryAcquire().ifTrue { + try { + readLock.withLock { + knownPostReplies.forEachIndexed { index, knownPostReply -> + configuration.getStringValue("KnownReplies/$index/ID").value = knownPostReply + } + configuration.getStringValue("KnownReplies/${knownPostReplies.size}/ID").value = null } - configuration.getStringValue("KnownReplies/${knownPostReplies.size}/ID").value = null + } catch (ce1: ConfigurationException) { + throw DatabaseException("Could not save database.", ce1) } - } catch (ce1: ConfigurationException) { - throw DatabaseException("Could not save database.", ce1) } } diff --git a/src/main/java/net/pterodactylus/sone/main/SonePlugin.java b/src/main/java/net/pterodactylus/sone/main/SonePlugin.java index 540f3ac..7723db1 100644 --- a/src/main/java/net/pterodactylus/sone/main/SonePlugin.java +++ b/src/main/java/net/pterodactylus/sone/main/SonePlugin.java @@ -22,7 +22,6 @@ import static java.util.logging.Logger.*; import java.util.logging.Logger; import java.util.logging.*; -import com.google.common.eventbus.*; import net.pterodactylus.sone.core.*; import net.pterodactylus.sone.fcp.*; import net.pterodactylus.sone.freenet.wot.*; @@ -35,8 +34,10 @@ import freenet.support.*; import freenet.support.api.*; import com.google.common.annotations.*; +import com.google.common.eventbus.*; import com.google.common.cache.*; import com.google.inject.*; +import com.google.inject.Module; import com.google.inject.name.*; import kotlin.jvm.functions.*; diff --git a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt index d01aeb4..409d18c 100644 --- a/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt +++ b/src/main/kotlin/net/pterodactylus/sone/core/DefaultElementLoader.kt @@ -23,9 +23,9 @@ class DefaultElementLoader(private val freenetInterface: FreenetInterface, ticke @Inject constructor(freenetInterface: FreenetInterface): this(freenetInterface, Ticker.systemTicker()) - private val loadingLinks: Cache = CacheBuilder.newBuilder().build() - private val failureCache: Cache = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build() - private val elementCache: Cache = CacheBuilder.newBuilder().build() + private val loadingLinks: Cache = CacheBuilder.newBuilder().build() + private val failureCache: Cache = CacheBuilder.newBuilder().ticker(ticker).expireAfterWrite(30, MINUTES).build() + private val elementCache: Cache = CacheBuilder.newBuilder().build() private val callback = object: FreenetInterface.BackgroundFetchCallback { override fun shouldCancel(uri: FreenetURI, mimeType: String, size: Long): Boolean { return (size > 2097152) || (!mimeType.startsWith("image/") && !mimeType.startsWith("text/html")) diff --git a/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt b/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt index 776d0c9..726ceb7 100644 --- a/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt +++ b/src/test/kotlin/net/pterodactylus/sone/core/DefaultElementLoaderTest.kt @@ -7,7 +7,7 @@ import net.pterodactylus.sone.core.FreenetInterface.BackgroundFetchCallback import net.pterodactylus.sone.test.capture import net.pterodactylus.sone.test.mock import org.hamcrest.MatcherAssert.assertThat -import org.hamcrest.Matchers.`is` +import org.hamcrest.Matchers.equalTo import org.hamcrest.Matchers.equalTo import org.junit.Test import org.mockito.ArgumentMatchers.any @@ -23,16 +23,6 @@ import java.util.concurrent.TimeUnit */ class DefaultElementLoaderTest { - companion object { - private const val IMAGE_ID = "KSK@gpl.png" - private val freenetURI = FreenetURI(IMAGE_ID) - private const val decomposedKey = "CHK@DCiVgTWW9nnWHJc9EVwtFJ6jAfBSVyy~rgiPvhUKbS4,mNY85V0x7dYcv7SnEYo1PCC6y2wNWMDNt-y9UWQx9fI,AAMC--8/fru%CC%88hstu%CC%88ck.jpg" - private const val normalizedKey = "CHK@DCiVgTWW9nnWHJc9EVwtFJ6jAfBSVyy~rgiPvhUKbS4,mNY85V0x7dYcv7SnEYo1PCC6y2wNWMDNt-y9UWQx9fI,AAMC--8/frühstück.jpg" - private const val textKey = "KSK@gpl.html" - private val sizeOkay = 2097152L - private val sizeNotOkay = sizeOkay + 1 - } - private val freenetInterface = mock() private val ticker = mock() private val elementLoader = DefaultElementLoader(freenetInterface, ticker) @@ -53,49 +43,49 @@ class DefaultElementLoaderTest { @Test fun `element loader returns loading element on first call`() { - assertThat(elementLoader.loadElement(IMAGE_ID).loading, `is`(true)) + assertThat(elementLoader.loadElement(IMAGE_ID).loading, equalTo(true)) } @Test fun `element loader does not cancel on image mime type with 2 mib size`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "image/png", sizeOkay), `is`(false)) + assertThat(callback.value.shouldCancel(freenetURI, "image/png", sizeOkay), equalTo(false)) } @Test fun `element loader does cancel on image mime type with more than 2 mib size`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "image/png", sizeNotOkay), `is`(true)) + assertThat(callback.value.shouldCancel(freenetURI, "image/png", sizeNotOkay), equalTo(true)) } @Test fun `element loader does cancel on audio mime type`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "audio/mpeg", sizeOkay), `is`(true)) + assertThat(callback.value.shouldCancel(freenetURI, "audio/mpeg", sizeOkay), equalTo(true)) } @Test fun `element loader does cancel on video mime type`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "video/mkv", sizeOkay), `is`(true)) + assertThat(callback.value.shouldCancel(freenetURI, "video/mkv", sizeOkay), equalTo(true)) } @Test fun `element loader does cancel on text mime type`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "text/plain", sizeOkay), `is`(true)) + assertThat(callback.value.shouldCancel(freenetURI, "text/plain", sizeOkay), equalTo(true)) } @Test fun `element loader does not cancel on text html mime type`() { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) - assertThat(callback.value.shouldCancel(freenetURI, "text/html", sizeOkay), `is`(false)) + assertThat(callback.value.shouldCancel(freenetURI, "text/html", sizeOkay), equalTo(false)) } @Test @@ -104,14 +94,14 @@ class DefaultElementLoaderTest { verify(freenetInterface).startFetch(eq(FreenetURI(decomposedKey)), callback.capture()) callback.value.loaded(FreenetURI(normalizedKey), "image/png", read("/static/images/unknown-image-0.png")) val linkedElement = elementLoader.loadElement(decomposedKey) - assertThat(linkedElement, `is`(LinkedElement(normalizedKey, properties = mapOf( + assertThat(linkedElement, equalTo(LinkedElement(normalizedKey, properties = mapOf( "type" to "image", "size" to 2451, "sizeHuman" to "2 KiB" )))) } @Test fun `element loader can extract description from description header`() { - elementLoader.loadElement(textKey) + elementLoader.loadElement(textKey) verify(freenetInterface).startFetch(eq(FreenetURI(textKey)), callback.capture()) callback.value.loaded(FreenetURI(textKey), "text/html; charset=UTF-8", read("element-loader.html")) val linkedElement = elementLoader.loadElement(textKey) @@ -126,7 +116,7 @@ class DefaultElementLoaderTest { @Test fun `element loader can extract description from first non-heading paragraph`() { - elementLoader.loadElement(textKey) + elementLoader.loadElement(textKey) verify(freenetInterface).startFetch(eq(FreenetURI(textKey)), callback.capture()) callback.value.loaded(FreenetURI(textKey), "text/html; charset=UTF-8", read("element-loader2.html")) val linkedElement = elementLoader.loadElement(textKey) @@ -141,7 +131,7 @@ class DefaultElementLoaderTest { @Test fun `element loader can not extract description if html is more complicated`() { - elementLoader.loadElement(textKey) + elementLoader.loadElement(textKey) verify(freenetInterface).startFetch(eq(FreenetURI(textKey)), callback.capture()) callback.value.loaded(FreenetURI(textKey), "text/html; charset=UTF-8", read("element-loader3.html")) val linkedElement = elementLoader.loadElement(textKey) @@ -156,7 +146,7 @@ class DefaultElementLoaderTest { @Test fun `element loader can not extract title if it is missing`() { - elementLoader.loadElement(textKey) + elementLoader.loadElement(textKey) verify(freenetInterface).startFetch(eq(FreenetURI(textKey)), callback.capture()) callback.value.loaded(FreenetURI(textKey), "text/html; charset=UTF-8", read("element-loader4.html")) val linkedElement = elementLoader.loadElement(textKey) @@ -174,7 +164,7 @@ class DefaultElementLoaderTest { elementLoader.loadElement(IMAGE_ID) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) callback.value.failed(freenetURI) - assertThat(elementLoader.loadElement(IMAGE_ID).failed, `is`(true)) + assertThat(elementLoader.loadElement(IMAGE_ID).failed, equalTo(true)) verify(freenetInterface).startFetch(eq(freenetURI), callback.capture()) } @@ -185,8 +175,8 @@ class DefaultElementLoaderTest { callback.value.failed(freenetURI) `when`(ticker.read()).thenReturn(TimeUnit.MINUTES.toNanos(31)) val linkedElement = elementLoader.loadElement(IMAGE_ID) - assertThat(linkedElement.failed, `is`(false)) - assertThat(linkedElement.loading, `is`(true)) + assertThat(linkedElement.failed, equalTo(false)) + assertThat(linkedElement.loading, equalTo(true)) verify(freenetInterface, times(2)).startFetch(eq(freenetURI), callback.capture()) } @@ -199,3 +189,11 @@ class DefaultElementLoaderTest { } ?: ByteArray(0) } + +private const val IMAGE_ID = "KSK@gpl.png" +private val freenetURI = FreenetURI(IMAGE_ID) +private const val decomposedKey = "CHK@DCiVgTWW9nnWHJc9EVwtFJ6jAfBSVyy~rgiPvhUKbS4,mNY85V0x7dYcv7SnEYo1PCC6y2wNWMDNt-y9UWQx9fI,AAMC--8/fru%CC%88hstu%CC%88ck.jpg" +private const val normalizedKey = "CHK@DCiVgTWW9nnWHJc9EVwtFJ6jAfBSVyy~rgiPvhUKbS4,mNY85V0x7dYcv7SnEYo1PCC6y2wNWMDNt-y9UWQx9fI,AAMC--8/frühstück.jpg" +private const val textKey = "KSK@gpl.html" +private const val sizeOkay = 2097152L +private const val sizeNotOkay = sizeOkay + 1 diff --git a/src/test/kotlin/net/pterodactylus/sone/database/memory/MemoryDatabaseTest.kt b/src/test/kotlin/net/pterodactylus/sone/database/memory/MemoryDatabaseTest.kt index 4d7a105..46796ab 100644 --- a/src/test/kotlin/net/pterodactylus/sone/database/memory/MemoryDatabaseTest.kt +++ b/src/test/kotlin/net/pterodactylus/sone/database/memory/MemoryDatabaseTest.kt @@ -417,6 +417,28 @@ class MemoryDatabaseTest { verify(configuration.getStringValue("KnownPosts/0/ID"), times(1)).value = null } + @Test + @Dirty("the rate limiter should be mocked") + fun `setting posts as knows twice in a row only saves the database once`() { + prepareConfigurationValues() + val post = mock() + whenever(post.id).thenReturn("post-id") + memoryDatabase.setPostKnown(post, true) + memoryDatabase.setPostKnown(post, true) + verify(configuration, times(1)).getStringValue("KnownPosts/1/ID") + } + + @Test + @Dirty("the rate limiter should be mocked") + fun `setting post replies as knows twice in a row only saves the database once`() { + prepareConfigurationValues() + val postReply = mock() + whenever(postReply.id).thenReturn("post-reply-id") + memoryDatabase.setPostReplyKnown(postReply, true) + memoryDatabase.setPostReplyKnown(postReply, true) + verify(configuration, times(1)).getStringValue("KnownReplies/1/ID") + } + } private const val SONE_ID = "sone"