1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Fix tls and crashes

This commit is contained in:
carbotaniuman 2022-02-17 06:27:43 +00:00
parent c7830161c1
commit d198c27de3
6 changed files with 70 additions and 43 deletions

View file

@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security ### Security
## [2.0.2] - 2022-02-16
### Removed
- [2022-02-16] Remove TLS 1.0 and 1.1 support [@carbotaniuman].
### Fixed
- [2022-02-16] Fix uncatched exceptions killing threads and not being logged [@carbotaniuman].
## [2.0.1] - 2021-05-27 ## [2.0.1] - 2021-05-27
### Added ### Added
- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman]. - [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
@ -396,7 +403,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed ### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD [Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...HEAD
[2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2
[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1 [2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0 [2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14

View file

@ -24,7 +24,6 @@ import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage import mdnet.cache.ImageStorage
import mdnet.data.Statistics import mdnet.data.Statistics
import mdnet.logging.error
import mdnet.logging.info import mdnet.logging.info
import mdnet.logging.warn import mdnet.logging.warn
import mdnet.metrics.DefaultMicrometerMetrics import mdnet.metrics.DefaultMicrometerMetrics
@ -188,7 +187,7 @@ class ServerManager(
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
LOGGER.error(e) { "Main loop failed" } LOGGER.warn(e) { "Main loop failed" }
} }
}, },
5, 5, TimeUnit.SECONDS 5, 5, TimeUnit.SECONDS

View file

@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.readValue
import mdnet.logging.info import mdnet.logging.info
import mdnet.logging.trace import mdnet.logging.trace
import mdnet.logging.warn
import org.apache.commons.io.file.PathUtils import org.apache.commons.io.file.PathUtils
import org.ktorm.database.Database import org.ktorm.database.Database
import org.ktorm.dsl.* import org.ktorm.dsl.*
@ -58,7 +59,7 @@ class ImageStorage(
private val cacheDirectory: Path, private val cacheDirectory: Path,
private val database: Database, private val database: Database,
autoPrune: Boolean = true autoPrune: Boolean = true
) { ) : AutoCloseable {
private val tempCacheDirectory = cacheDirectory.resolve("tmp") private val tempCacheDirectory = cacheDirectory.resolve("tmp")
private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2) private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2)
@ -89,36 +90,43 @@ class ImageStorage(
evictor.scheduleWithFixedDelay( evictor.scheduleWithFixedDelay(
{ {
val toUpdate = HashSet<String>() try {
queue.drainTo(toUpdate) val toUpdate = HashSet<String>()
val now = Instant.now() queue.drainTo(toUpdate)
val now = Instant.now()
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" } LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
synchronized(database) { synchronized(database) {
database.batchUpdate(DbImage) { database.batchUpdate(DbImage) {
for (id in toUpdate) { for (id in toUpdate) {
item { item {
set(DbImage.accessed, now) set(DbImage.accessed, now)
where { where {
DbImage.id eq id DbImage.id eq id
}
} }
} }
} }
} }
calculateSize()
} catch (e: Exception) {
LOGGER.warn(e) { "Error updating LRU $this" }
} }
calculateSize()
}, },
1, 1, TimeUnit.MINUTES 30, 30, TimeUnit.SECONDS
) )
// evict LRU cache every 3 minutes
if (autoPrune) { if (autoPrune) {
evictor.scheduleWithFixedDelay( evictor.scheduleWithFixedDelay(
{ {
calculateSize() try {
pruneImages() calculateSize()
pruneImages()
} catch (e: Exception) {
LOGGER.warn(e) { "Error pruning images" }
}
}, },
0, 3, TimeUnit.MINUTES 0, 1, TimeUnit.MINUTES
) )
} }
} }
@ -255,7 +263,7 @@ class ImageStorage(
} }
} }
fun close() { override fun close() {
evictor.shutdown() evictor.shutdown()
evictor.awaitTermination(10, TimeUnit.SECONDS) evictor.awaitTermination(10, TimeUnit.SECONDS)
} }

View file

@ -175,7 +175,7 @@ class Netty(
val certs = getX509Certs(tls.certificate) val certs = getX509Certs(tls.certificate)
val sslContext = SslContextBuilder val sslContext = SslContextBuilder
.forServer(getPrivateKey(tls.privateKey), certs) .forServer(getPrivateKey(tls.privateKey), certs)
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1") .protocols("TLSv1.3", "TLSv1.2")
.build() .build()
val bootstrap = ServerBootstrap() val bootstrap = ServerBootstrap()

View file

@ -39,11 +39,13 @@ class ImageStorageTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest override fun isolationMode() = IsolationMode.InstancePerTest
init { init {
val imageStorage = ImageStorage( val imageStorage = autoClose(
maxSize = 5, ImageStorage(
cacheDirectory = tempdir().toPath(), maxSize = 5,
database = Database.connect("jdbc:sqlite:${tempfile()}"), cacheDirectory = tempdir().toPath(),
autoPrune = false, database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
) )
val testMeta = ImageMetadata("a", "a", 123) val testMeta = ImageMetadata("a", "a", 123)
@ -81,7 +83,8 @@ class ImageStorageTest : FreeSpec() {
writer.stream.write(ByteArray(12)) writer.stream.write(ByteArray(12))
writer.abort() writer.abort()
"should not update size" { "should not update size even if calculated" {
imageStorage.calculateSize()
imageStorage.size.shouldBeZero() imageStorage.size.shouldBeZero()
} }
} }
@ -157,14 +160,17 @@ class ImageStorageSlowTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest override fun isolationMode() = IsolationMode.InstancePerTest
init { init {
val imageStorage = ImageStorage( val imageStorage = autoClose(
maxSize = 4097, ImageStorage(
cacheDirectory = tempdir().toPath(), maxSize = 4097,
database = Database.connect("jdbc:sqlite:${tempfile()}"), cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
)
) )
"autoPrune" - { "autoPrune" - {
"should update size eventually" { "should update size eventually" {
println("1 - $imageStorage")
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096)) val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096))
writer.shouldNotBeNull() writer.shouldNotBeNull()
@ -177,6 +183,7 @@ class ImageStorageSlowTest : FreeSpec() {
} }
"should prune if insufficient size eventually" { "should prune if insufficient size eventually" {
println("2 - $imageStorage")
imageStorage.maxSize = 10000 imageStorage.maxSize = 10000
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123)) val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123))
@ -185,6 +192,7 @@ class ImageStorageSlowTest : FreeSpec() {
writer.stream.write(ByteArray(8192)) writer.stream.write(ByteArray(8192))
writer.commit(8192).shouldBeTrue() writer.commit(8192).shouldBeTrue()
imageStorage.calculateSize()
eventually(Duration.minutes(5)) { eventually(Duration.minutes(5)) {
imageStorage.size.shouldBeZero() imageStorage.size.shouldBeZero()
} }

View file

@ -116,11 +116,13 @@ class ImageServerTest : FreeSpec() {
} }
"with real cache" - { "with real cache" - {
val storage = ImageStorage( val storage = autoClose(
maxSize = 100000, ImageStorage(
cacheDirectory = tempdir().toPath(), maxSize = 100000,
database = Database.connect("jdbc:sqlite:${tempfile()}"), cacheDirectory = tempdir().toPath(),
autoPrune = false, database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
) )
val server = ImageServer( val server = ImageServer(
@ -161,11 +163,13 @@ class ImageServerTest : FreeSpec() {
"failed upstream responses" - { "failed upstream responses" - {
val client = mockk<HttpHandler>() val client = mockk<HttpHandler>()
val storage = ImageStorage( val storage = autoClose(
maxSize = 100000, ImageStorage(
cacheDirectory = tempdir().toPath(), maxSize = 100000,
database = Database.connect("jdbc:sqlite:${tempfile()}"), cacheDirectory = tempdir().toPath(),
autoPrune = false, database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
) )
val server = ImageServer( val server = ImageServer(