1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Try and fix contention issues

This commit is contained in:
carbotaniuman 2023-04-06 21:27:06 +00:00
parent c9a5548770
commit ac246e5449
4 changed files with 50 additions and 27 deletions

View file

@ -6,7 +6,7 @@ plugins {
id "application"
id "com.github.johnrengelman.shadow" version "7.0.0"
id "com.diffplug.spotless" version "5.8.2"
id "net.afanasev.sekret" version "0.1.1-RC3"
id "net.afanasev.sekret" version "0.1.1"
id "com.palantir.git-version" version "0.12.3"
}
@ -28,7 +28,7 @@ configurations {
dependencies {
implementation "org.jetbrains.kotlin:kotlin-reflect"
compileOnly group: "net.afanasev", name: "sekret-annotation", version: "0.1.1-RC3"
compileOnly group: "net.afanasev", name: "sekret-annotation", version: "0.1.1"
implementation group: "commons-io", name: "commons-io", version: "2.11.0"
implementation group: "org.apache.commons", name: "commons-compress", version: "1.21"
@ -66,7 +66,7 @@ dependencies {
testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
testImplementation "io.kotest:kotest-assertions-core:$kotest_version"
testImplementation "io.mockk:mockk:1.12.2"
testImplementation "io.mockk:mockk:1.12.3"
}
tasks.withType(Test) {

View file

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View file

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 31
const val CLIENT_BUILD = 32
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

View file

@ -36,6 +36,7 @@ import java.sql.SQLException
import java.time.Instant
import java.util.UUID
import java.util.concurrent.*
import java.util.concurrent.locks.ReentrantLock
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class ImageMetadata(
@ -61,6 +62,7 @@ class ImageStorage(
autoPrune: Boolean = true
) : AutoCloseable {
private val tempCacheDirectory = cacheDirectory.resolve("tmp")
private val databaseLock = ReentrantLock()
private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(1)
private val queue = LinkedBlockingQueue<String>(1000)
@ -96,7 +98,9 @@ class ImageStorage(
val now = Instant.now()
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
synchronized(database) {
if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
try {
database.batchUpdate(DbImage) {
for (id in toUpdate) {
item {
@ -107,6 +111,11 @@ class ImageStorage(
}
}
}
} finally {
databaseLock.unlock()
}
} else {
LOGGER.warn { "High contention for database lock, bailing LRU update" }
}
calculateSize()
} catch (e: Exception) {
@ -238,14 +247,24 @@ class ImageStorage(
)
Files.deleteIfExists(path)
} catch (e: IOException) {
// a failure means the image did not exist
} finally {
synchronized(database) {
} catch (_: IOException) {
}
// it is safe, but not optimal, for the
// DB write to fail after we've grabbed the file,
// as that just inflates the count.
// it will get resolved when the file gets grabbed again,
// or if the cache gets pruned.
if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
try {
database.delete(DbImage) {
DbImage.id eq id
}
} finally {
databaseLock.unlock()
}
} else {
LOGGER.warn { "High contention for database lock, bailing image delete write" }
}
}
@ -319,14 +338,13 @@ class ImageStorage(
Files.createDirectories(getPath(id).parent)
if (databaseLock.tryLock(500, TimeUnit.MILLISECONDS)) {
try {
synchronized(database) {
database.insert(DbImage) {
set(DbImage.id, id)
set(DbImage.accessed, Instant.now())
set(DbImage.size, metadataSize + bytes)
}
}
} catch (e: SQLException) {
// someone got to us before this (TOCTOU)
// there are 2 situations here
@ -336,6 +354,11 @@ class ImageStorage(
// the other is that we have raced and the other
// is about to write the file
// we handle this below
} finally {
databaseLock.unlock()
}
} else {
LOGGER.warn { "High contention for database lock, bailing DB write" }
}
try {