From d198c27de3aa6cfd549a84abbce74e88046b6e20 Mon Sep 17 00:00:00 2001 From: carbotaniuman Date: Thu, 17 Feb 2022 06:27:43 +0000 Subject: [PATCH] Fix tls and crashes --- CHANGELOG.md | 10 +++- src/main/kotlin/mdnet/ServerManager.kt | 3 +- src/main/kotlin/mdnet/cache/ImageStorage.kt | 46 +++++++++++-------- .../kotlin/mdnet/netty/ApplicationNetty.kt | 2 +- .../kotlin/mdnet/cache/ImageStorageTest.kt | 28 +++++++---- .../kotlin/mdnet/server/ImageServerTest.kt | 24 ++++++---- 6 files changed, 70 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303cbfc..9d15ac4 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [2.0.2] - 2022-02-16 +### Removed +- [2022-02-16] Remove TLS 1.0 and 1.1 support [@carbotaniuman]. + +### Fixed +- [2022-02-16] Fix uncatched exceptions killing threads and not being logged [@carbotaniuman]. + ## [2.0.1] - 2021-05-27 ### Added - [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman]. @@ -396,7 +403,8 @@ This release contains many breaking changes! Of note are the changes to the cach ### Fixed - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. -[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD +[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...HEAD +[2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2 [2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1 [2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14 diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index b024523..42d39df 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -24,7 +24,6 @@ import io.micrometer.prometheus.PrometheusConfig import io.micrometer.prometheus.PrometheusMeterRegistry import mdnet.cache.ImageStorage import mdnet.data.Statistics -import mdnet.logging.error import mdnet.logging.info import mdnet.logging.warn import mdnet.metrics.DefaultMicrometerMetrics @@ -188,7 +187,7 @@ class ServerManager( } } } catch (e: Exception) { - LOGGER.error(e) { "Main loop failed" } + LOGGER.warn(e) { "Main loop failed" } } }, 5, 5, TimeUnit.SECONDS diff --git a/src/main/kotlin/mdnet/cache/ImageStorage.kt b/src/main/kotlin/mdnet/cache/ImageStorage.kt index ce96495..5cb60c1 100644 --- a/src/main/kotlin/mdnet/cache/ImageStorage.kt +++ b/src/main/kotlin/mdnet/cache/ImageStorage.kt @@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import mdnet.logging.info import mdnet.logging.trace +import mdnet.logging.warn import org.apache.commons.io.file.PathUtils import org.ktorm.database.Database import org.ktorm.dsl.* @@ -58,7 +59,7 @@ class ImageStorage( private val cacheDirectory: Path, private val database: Database, autoPrune: Boolean = true -) { +) : AutoCloseable { private val tempCacheDirectory = cacheDirectory.resolve("tmp") private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2) @@ -89,36 +90,43 @@ class ImageStorage( evictor.scheduleWithFixedDelay( { - val toUpdate = HashSet() - queue.drainTo(toUpdate) - val now = Instant.now() + try { + val toUpdate = HashSet() + queue.drainTo(toUpdate) + val now = Instant.now() - LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" } - synchronized(database) { - database.batchUpdate(DbImage) { - for (id in toUpdate) { - item { - set(DbImage.accessed, now) - where { - DbImage.id eq id + LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" } + synchronized(database) { + database.batchUpdate(DbImage) { + for (id in toUpdate) { + item { + set(DbImage.accessed, now) + where { + DbImage.id eq id + } } } } } + calculateSize() + } catch (e: Exception) { + LOGGER.warn(e) { "Error updating LRU $this" } } - calculateSize() }, - 1, 1, TimeUnit.MINUTES + 30, 30, TimeUnit.SECONDS ) - // evict LRU cache every 3 minutes if (autoPrune) { evictor.scheduleWithFixedDelay( { - calculateSize() - pruneImages() + try { + calculateSize() + pruneImages() + } catch (e: Exception) { + LOGGER.warn(e) { "Error pruning images" } + } }, - 0, 3, TimeUnit.MINUTES + 0, 1, TimeUnit.MINUTES ) } } @@ -255,7 +263,7 @@ class ImageStorage( } } - fun close() { + override fun close() { evictor.shutdown() evictor.awaitTermination(10, TimeUnit.SECONDS) } diff --git a/src/main/kotlin/mdnet/netty/ApplicationNetty.kt b/src/main/kotlin/mdnet/netty/ApplicationNetty.kt index b208f1a..830a3df 100644 --- a/src/main/kotlin/mdnet/netty/ApplicationNetty.kt +++ b/src/main/kotlin/mdnet/netty/ApplicationNetty.kt @@ -175,7 +175,7 @@ class Netty( val certs = getX509Certs(tls.certificate) val sslContext = SslContextBuilder .forServer(getPrivateKey(tls.privateKey), certs) - .protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1") + .protocols("TLSv1.3", "TLSv1.2") .build() val bootstrap = ServerBootstrap() diff --git a/src/test/kotlin/mdnet/cache/ImageStorageTest.kt b/src/test/kotlin/mdnet/cache/ImageStorageTest.kt index 67db2eb..80bf8d8 100644 --- a/src/test/kotlin/mdnet/cache/ImageStorageTest.kt +++ b/src/test/kotlin/mdnet/cache/ImageStorageTest.kt @@ -39,11 +39,13 @@ class ImageStorageTest : FreeSpec() { override fun isolationMode() = IsolationMode.InstancePerTest init { - val imageStorage = ImageStorage( - maxSize = 5, - cacheDirectory = tempdir().toPath(), - database = Database.connect("jdbc:sqlite:${tempfile()}"), - autoPrune = false, + val imageStorage = autoClose( + ImageStorage( + maxSize = 5, + cacheDirectory = tempdir().toPath(), + database = Database.connect("jdbc:sqlite:${tempfile()}"), + autoPrune = false, + ) ) val testMeta = ImageMetadata("a", "a", 123) @@ -81,7 +83,8 @@ class ImageStorageTest : FreeSpec() { writer.stream.write(ByteArray(12)) writer.abort() - "should not update size" { + "should not update size even if calculated" { + imageStorage.calculateSize() imageStorage.size.shouldBeZero() } } @@ -157,14 +160,17 @@ class ImageStorageSlowTest : FreeSpec() { override fun isolationMode() = IsolationMode.InstancePerTest init { - val imageStorage = ImageStorage( - maxSize = 4097, - cacheDirectory = tempdir().toPath(), - database = Database.connect("jdbc:sqlite:${tempfile()}"), + val imageStorage = autoClose( + ImageStorage( + maxSize = 4097, + cacheDirectory = tempdir().toPath(), + database = Database.connect("jdbc:sqlite:${tempfile()}"), + ) ) "autoPrune" - { "should update size eventually" { + println("1 - $imageStorage") val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096)) writer.shouldNotBeNull() @@ -177,6 +183,7 @@ class ImageStorageSlowTest : FreeSpec() { } "should prune if insufficient size eventually" { + println("2 - $imageStorage") imageStorage.maxSize = 10000 val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123)) @@ -185,6 +192,7 @@ class ImageStorageSlowTest : FreeSpec() { writer.stream.write(ByteArray(8192)) writer.commit(8192).shouldBeTrue() + imageStorage.calculateSize() eventually(Duration.minutes(5)) { imageStorage.size.shouldBeZero() } diff --git a/src/test/kotlin/mdnet/server/ImageServerTest.kt b/src/test/kotlin/mdnet/server/ImageServerTest.kt index 60567ba..84bb667 100644 --- a/src/test/kotlin/mdnet/server/ImageServerTest.kt +++ b/src/test/kotlin/mdnet/server/ImageServerTest.kt @@ -116,11 +116,13 @@ class ImageServerTest : FreeSpec() { } "with real cache" - { - val storage = ImageStorage( - maxSize = 100000, - cacheDirectory = tempdir().toPath(), - database = Database.connect("jdbc:sqlite:${tempfile()}"), - autoPrune = false, + val storage = autoClose( + ImageStorage( + maxSize = 100000, + cacheDirectory = tempdir().toPath(), + database = Database.connect("jdbc:sqlite:${tempfile()}"), + autoPrune = false, + ) ) val server = ImageServer( @@ -161,11 +163,13 @@ class ImageServerTest : FreeSpec() { "failed upstream responses" - { val client = mockk() - val storage = ImageStorage( - maxSize = 100000, - cacheDirectory = tempdir().toPath(), - database = Database.connect("jdbc:sqlite:${tempfile()}"), - autoPrune = false, + val storage = autoClose( + ImageStorage( + maxSize = 100000, + cacheDirectory = tempdir().toPath(), + database = Database.connect("jdbc:sqlite:${tempfile()}"), + autoPrune = false, + ) ) val server = ImageServer(