Merge branch 'update-everything' into 'master'

Update all the deps

See merge request mangadex-pub/mangadex_at_home!103

.gitlab-ci.yml

@@ -11,11 +11,13 @@ Gradle Build:
     - branches
     - tags
     - merge_requests
-
   before_script:
     - export VERSION="${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}"
   script:
     - ./gradlew build
+  variables:
+    KUBERNETES_MEMORY_REQUEST: 3Gi
+    KUBERNETES_MEMORY_LIMIT: 3Gi
   artifacts:
     name: "mangadex_at_home"
     paths:
@@ -60,6 +62,7 @@ Docker Build:
     - docker build . -t $CI_REGISTRY_IMAGE:$BASE_TAG
     - docker push $CI_REGISTRY_IMAGE:$BASE_TAG
 
+
 .docker_push: &docker_push
   image: docker:20.10.8
   services:
@@ -71,11 +74,13 @@ Docker Build:
   before_script:
     - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin ${CI_REGISTRY}
     - export BASE_TAG="git-$CI_COMMIT_SHORT_SHA"
+    - export SHORT_TAG="$(echo $CI_COMMIT_TAG | cut -d "." -f1)"
 
   script:
     - docker pull $CI_REGISTRY_IMAGE:$BASE_TAG
     - docker tag $CI_REGISTRY_IMAGE:$BASE_TAG $CI_REGISTRY_IMAGE:$NEW_TAG
-    - docker push $CI_REGISTRY_IMAGE:$NEW_TAG
+    - docker tag $CI_REGISTRY_IMAGE:$BASE_TAG $CI_REGISTRY_IMAGE:$SHORT_TAG
+    - docker push $CI_REGISTRY_IMAGE --all-tags
 
 Push Latest:
   <<: *docker_push

CHANGELOG.md

@@ -17,6 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [2.0.4] - 2023-08-07
+### Changed
+- [2023-08-07] Updated dependencies [@carbotaniuman].
+- [2023-04-06] Fixed DB contention issues [@carbotaniuman].
+- [2023-04-06] Make errors more useful [@carbotaniuman].
+
 ## [2.0.3] - 2022-02-17
 ### Changed
 - [2022-02-17] Updated dependencies [@carbotaniuman].
@@ -411,7 +417,8 @@ This release contains many breaking changes! Of note are the changes to the cach
 ### Fixed
 - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
 
-[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.3...HEAD
+[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.4...HEAD
+[2.0.4]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.3...2.0.4
 [2.0.3]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...2.0.3
 [2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2
 [2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1

build.gradle

@@ -1,8 +1,10 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
     id "jacoco"
     id "java"
-    id "org.jetbrains.kotlin.jvm" version "1.6.0"
-    id "org.jetbrains.kotlin.kapt" version "1.6.0"
+    id "org.jetbrains.kotlin.jvm" version "1.8.0"
+    id "org.jetbrains.kotlin.kapt" version "1.8.0"
     id "application"
     id "com.github.johnrengelman.shadow" version "7.0.0"
     id "com.diffplug.spotless" version "5.8.2"
@@ -28,18 +30,16 @@ configurations {
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-reflect"
 
-    compileOnly group: "net.afanasev", name: "sekret-annotation", version: "0.1.1-RC3"
-
     implementation group: "commons-io", name: "commons-io", version: "2.11.0"
-    implementation group: "org.apache.commons", name: "commons-compress", version: "1.21"
-    implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
+    implementation group: "org.apache.commons", name: "commons-compress", version: "1.22"
+    implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.6"
 
     implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.8.3"
-    implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
+    implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.16.1"
 
-    implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.19.3.0")
-    implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.13.1")
-    implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.74.Final")
+    implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.41.3.0")
+    implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.14.2")
+    implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.91.Final")
 
     implementation group: "org.http4k", name: "http4k-core"
     implementation group: "org.http4k", name: "http4k-resilience4j"
@@ -52,12 +52,12 @@ dependencies {
     implementation group: "org.http4k", name: "http4k-server-netty"
     implementation group: "io.netty", name: "netty-codec-haproxy"
     implementation group: "io.netty", name: "netty-transport-native-epoll", classifier: "linux-x86_64"
-    implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.11.Final", classifier: "linux-x86_64"
+    implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.19.Final", classifier: "linux-x86_64"
     testImplementation group: "org.http4k", name: "http4k-testing-kotest"
-    runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.48.Final"
+    runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.59.Final"
 
     implementation group: "com.zaxxer", name: "HikariCP", version: "4.0.3"
-    implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.34.0"
+    implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.41.2.1"
     implementation "org.ktorm:ktorm-core:$ktorm_version"
     implementation "org.ktorm:ktorm-jackson:$ktorm_version"
 
@@ -66,10 +66,10 @@ dependencies {
 
     testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
     testImplementation "io.kotest:kotest-assertions-core:$kotest_version"
-    testImplementation "io.mockk:mockk:1.12.2"
+    testImplementation "io.mockk:mockk:1.13.4"
 }
 
-tasks.withType(Test) {
+tasks.withType(Test).configureEach {
     useJUnitPlatform()
     javaLauncher = javaToolchains.launcherFor {
         languageVersion = JavaLanguageVersion.of(8)
@@ -91,14 +91,14 @@ kapt {
 }
 
 java {
-    sourceCompatibility = JavaVersion.VERSION_1_8
-    targetCompatibility = JavaVersion.VERSION_1_8
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(8))
+    }
 }
 
-tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
-    kotlinOptions {
+tasks.withType(KotlinCompile).configureEach {
+    compilerOptions {
         freeCompilerArgs = ["-Xjsr305=strict"]
-        jvmTarget = "1.8"
     }
 }
 

docker/README.md

@@ -12,6 +12,8 @@ Once installed, you can check that it works by opening a command prompt and runn
 
 ## Run as a standalone container
 
+*Note* Changes to `the docker-compose.yml` are coming, and as such, this instruction page will get reworked a bit.
+
 Use either a specific image, preferrably the [latest image published](https://gitlab.com/mangadex-pub/mangadex_at_home/container_registry/1200259)
 
 > While it might work, using `registry.gitlab.com/mangadex-pub/mangadex_at_home:latest` is a bad idea as we do not guarantee forward-compatibility

docker/docker-compose.yml

@@ -4,7 +4,7 @@ services:
 
   mangadex-at-home:
     container_name: mangadex-at-home
-    image: "registry.gitlab.com/mangadex-pub/mangadex_at_home:<version>"
+    image: "registry.gitlab.com/mangadex-pub/mangadex_at_home:2"
     ports:
       - 443:443
     volumes:
@@ -12,7 +12,6 @@ services:
       - ./data/cache/:/mangahome/data/
     environment:
       JAVA_TOOL_OPTIONS: "-Xms1G -Xmx1G -XX:+UseShenandoahGC -Xss512K"
-    privileged: true
     command: [
         "bash",
         "-c",
@@ -31,7 +30,7 @@ services:
 
   prometheus:
     container_name: prometheus
-    image: prom/prometheus:v2.24.1
+    image: prom/prometheus:v2.34.0
     user: "root"
     group_add:
       - 0
@@ -50,7 +49,7 @@ services:
 
   grafana:
     container_name: grafana
-    image: grafana/grafana:7.4.0
+    image: grafana/grafana:8.4.3
     user: "root"
     group_add:
       - 0

gradle.properties

@@ -1,3 +1,3 @@
-kotest_version=5.1.0
-ktorm_version=3.4.1
-picocli_version=4.6.3
+kotest_version=5.5.5
+ktorm_version=3.6.0
+picocli_version=4.7.1

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -72,7 +72,7 @@ case "`uname`" in
   Darwin* )
     darwin=true
     ;;
-  MINGW* )
+  MSYS* | MINGW* )
     msys=true
     ;;
   NONSTOP* )
@@ -130,7 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath

gradlew.bat

@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -54,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -64,21 +64,6 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
@@ -86,7 +71,7 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

src/main/kotlin/mdnet/BackendApi.kt

@@ -124,7 +124,7 @@ class BackendApi(private val settings: ClientSettings) {
                 try {
                     PING_FAILURE_LENS(response)
                 } catch (e: LensFailure) {
-                    PingFailure(response.status.code, response.status.description)
+                    PingFailure(response.status.code, response.bodyString())
                 }
             }
         }

src/main/kotlin/mdnet/Constants.kt

@@ -21,7 +21,7 @@ package mdnet
 import java.time.Duration
 
 object Constants {
-    const val CLIENT_BUILD = 31
+    const val CLIENT_BUILD = 32
 
     @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
 

src/main/kotlin/mdnet/cache/ImageStorage.kt

@@ -36,6 +36,7 @@ import java.sql.SQLException
 import java.time.Instant
 import java.util.UUID
 import java.util.concurrent.*
+import java.util.concurrent.locks.ReentrantLock
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class ImageMetadata(
@@ -61,6 +62,7 @@ class ImageStorage(
     autoPrune: Boolean = true
 ) : AutoCloseable {
     private val tempCacheDirectory = cacheDirectory.resolve("tmp")
+    private val databaseLock = ReentrantLock()
 
     private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(1)
     private val queue = LinkedBlockingQueue<String>(1000)
@@ -93,20 +95,32 @@ class ImageStorage(
                 try {
                     val toUpdate = HashSet<String>()
                     queue.drainTo(toUpdate)
+
+                    if (toUpdate.isEmpty()) {
+                        LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
+                    } else {
+                        LOGGER.info { "Skipping empty LRU update" }
+                    }
+
                     val now = Instant.now()
 
-                    LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
-                    synchronized(database) {
-                        database.batchUpdate(DbImage) {
-                            for (id in toUpdate) {
-                                item {
-                                    set(DbImage.accessed, now)
-                                    where {
-                                        DbImage.id eq id
+                    if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
+                        try {
+                            database.batchUpdate(DbImage) {
+                                for (id in toUpdate) {
+                                    item {
+                                        set(DbImage.accessed, now)
+                                        where {
+                                            DbImage.id eq id
+                                        }
                                     }
                                 }
                             }
+                        } finally {
+                            databaseLock.unlock()
                         }
+                    } else {
+                        LOGGER.warn { "High contention for database lock, bailing LRU update" }
                     }
                     calculateSize()
                 } catch (e: Exception) {
@@ -238,14 +252,24 @@ class ImageStorage(
             )
 
             Files.deleteIfExists(path)
-        } catch (e: IOException) {
-            // a failure means the image did not exist
-        } finally {
-            synchronized(database) {
+        } catch (_: IOException) {
+        }
+
+        // it is safe, but not optimal, for the
+        // DB write to fail after we've grabbed the file,
+        // as that just inflates the count.
+        // it will get resolved when the file gets grabbed again,
+        // or if the cache gets pruned.
+        if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
+            try {
                 database.delete(DbImage) {
                     DbImage.id eq id
                 }
+            } finally {
+                databaseLock.unlock()
             }
+        } else {
+            LOGGER.warn { "High contention for database lock, bailing image delete write" }
         }
     }
 
@@ -319,23 +343,27 @@ class ImageStorage(
 
             Files.createDirectories(getPath(id).parent)
 
-            try {
-                synchronized(database) {
+            if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
+                try {
                     database.insert(DbImage) {
                         set(DbImage.id, id)
                         set(DbImage.accessed, Instant.now())
                         set(DbImage.size, metadataSize + bytes)
                     }
+                } catch (e: SQLException) {
+                    // someone got to us before this (TOCTOU)
+                    // there are 2 situations here
+                    // one is that the
+                    // other write died in between writing the DB and
+                    // moving the file
+                    // the other is that we have raced and the other
+                    // is about to write the file
+                    // we handle this below
+                } finally {
+                    databaseLock.unlock()
                 }
-            } catch (e: SQLException) {
-                // someone got to us before this (TOCTOU)
-                // there are 2 situations here
-                // one is that the
-                // other write died in between writing the DB and
-                // moving the file
-                // the other is that we have raced and the other
-                // is about to write the file
-                // we handle this below
+            } else {
+                LOGGER.warn { "High contention for database lock, bailing DB write" }
             }
 
             try {

src/main/kotlin/mdnet/settings/ClientSettings.kt

@@ -20,7 +20,6 @@ package mdnet.settings
 
 import com.fasterxml.jackson.databind.PropertyNamingStrategies
 import com.fasterxml.jackson.databind.annotation.JsonNaming
-import net.afanasev.sekret.Secret
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class ClientSettings(
@@ -33,7 +32,7 @@ data class ClientSettings(
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class ServerSettings(
-    @field:Secret val secret: String,
+    val secret: String,
     val externalPort: Int = 0,
     val gracefulShutdownWaitSeconds: Int = 60,
     val hostname: String = "0.0.0.0",
@@ -44,7 +43,11 @@ data class ServerSettings(
     val port: Int = 443,
     val threads: Int = 0,
     val enableProxyProtocol: Boolean = false,
-)
+) {
+    override fun toString(): String {
+        return "ServerSettings(secret=<redacted>, externalPort=$externalPort, gracefulShutdownWaitSeconds=$gracefulShutdownWaitSeconds, hostname='$hostname', maxKilobitsPerSecond=$maxKilobitsPerSecond, externalMaxKilobitsPerSecond=$externalMaxKilobitsPerSecond, maxMebibytesPerHour=$maxMebibytesPerHour, externalIp=$externalIp, port=$port, threads=$threads, enableProxyProtocol=$enableProxyProtocol)"
+    }
+}
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class DevSettings(
@@ -57,5 +60,9 @@ data class DevSettings(
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class MetricsSettings(
     val enableGeoip: Boolean = false,
-    @field:Secret val geoipLicenseKey: String = "none"
-)
+    val geoipLicenseKey: String = "none"
+) {
+    override fun toString(): String {
+        return "MetricsSettings(enableGeoip=$enableGeoip, geoipLicenseKey=<redacted>)"
+    }
+}

src/main/kotlin/mdnet/settings/PingResult.kt

@@ -20,7 +20,6 @@ package mdnet.settings
 
 import com.fasterxml.jackson.databind.PropertyNamingStrategies
 import com.fasterxml.jackson.databind.annotation.JsonNaming
-import net.afanasev.sekret.Secret
 import org.http4k.core.Uri
 
 sealed class PingResult
@@ -37,12 +36,13 @@ data class RemoteSettings(
     val latestBuild: Int,
     val url: Uri,
     val clientId: String,
-    @field:Secret val tokenKey: ByteArray,
+    val tokenKey: ByteArray,
     val compromised: Boolean,
     val paused: Boolean,
     val disableTokens: Boolean = false,
     val tls: TlsCert?
 ) : PingResult() {
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (javaClass != other?.javaClass) return false
@@ -74,11 +74,19 @@ data class RemoteSettings(
         result = 31 * result + (tls?.hashCode() ?: 0)
         return result
     }
+
+    override fun toString(): String {
+        return "RemoteSettings(imageServer=$imageServer, latestBuild=$latestBuild, url=$url, clientId='$clientId', tokenKey=<redacted>, compromised=$compromised, paused=$paused, disableTokens=$disableTokens, tls=$tls)"
+    }
 }
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class TlsCert(
     val createdAt: String,
-    @field:Secret val privateKey: String,
-    @field:Secret val certificate: String
-)
+    val privateKey: String,
+    val certificate: String
+) {
+    override fun toString(): String {
+        return "TlsCert(createdAt='$createdAt', privateKey=<redacted>, certificate=<redacted>)"
+    }
+}

src/main/kotlin/mdnet/settings/SettingsRequest.kt

@@ -20,20 +20,27 @@ package mdnet.settings
 
 import com.fasterxml.jackson.databind.PropertyNamingStrategies
 import com.fasterxml.jackson.databind.annotation.JsonNaming
-import net.afanasev.sekret.Secret
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class SettingsRequest(
-    @field:Secret val secret: String,
+    val secret: String,
     val ipAddress: String?,
     val port: Int,
     val diskSpace: Long,
     val networkSpeed: Long,
     val buildVersion: Int,
     val tlsCreatedAt: String?,
-)
+) {
+    override fun toString(): String {
+        return "SettingsRequest(secret=<redacted>, ipAddress=$ipAddress, port=$port, diskSpace=$diskSpace, networkSpeed=$networkSpeed, buildVersion=$buildVersion, tlsCreatedAt=$tlsCreatedAt)"
+    }
+}
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class LogoutRequest(
-    @field:Secret val secret: String,
-)
+    val secret: String,
+) {
+    override fun toString(): String {
+        return "LogoutRequest(secret=<redacted>)"
+    }
+}

src/test/kotlin/mdnet/cache/ImageStorageTest.kt

@@ -32,7 +32,7 @@ import io.kotest.matchers.nulls.shouldNotBeNull
 import io.kotest.matchers.shouldBe
 import org.apache.commons.io.IOUtils
 import org.ktorm.database.Database
-import kotlin.time.Duration
+import kotlin.time.Duration.Companion.minutes
 import kotlin.time.ExperimentalTime
 
 class ImageStorageTest : FreeSpec() {
@@ -177,7 +177,7 @@ class ImageStorageSlowTest : FreeSpec() {
                 writer.stream.write(ByteArray(4096))
                 writer.commit(4096).shouldBeTrue()
 
-                eventually(Duration.minutes(5)) {
+                eventually(5.minutes) {
                     imageStorage.size.shouldBeGreaterThan(0)
                 }
             }
@@ -193,7 +193,7 @@ class ImageStorageSlowTest : FreeSpec() {
                 writer.commit(8192).shouldBeTrue()
 
                 imageStorage.calculateSize()
-                eventually(Duration.minutes(5)) {
+                eventually(5.minutes) {
                     imageStorage.size.shouldBeZero()
                 }
             }