This commit is contained in:
carbotaniuman 2021-03-11 20:09:14 +00:00
parent 8178f1b0d6
commit f9cf5729b4
16 changed files with 139 additions and 196 deletions

View File

@ -14,15 +14,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
### Fixed
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
### Security
## [2.0.0] - 2021-03-11
### Changed
- [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
- [2021-03-11] Tune connection pool [@carbotaniuman].
### Fixed
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
- [2021-03-11] Throw exceptions less frequently [@carbotaniuman].
- [2021-03-11] Fix various Netty issues [@carbotaniuman].
- [2021-03-11] Don't log IOUring and Epoll errors [@carbotaniuman].
- [2021-03-11] Ignore IPV6 when pinging [@carbotaniuman].
## [2.0.0-rc14] - 2021-03-02
### Changed
- [2021-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2021-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2021-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2021-03-02] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2021-03-02] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2021-03-02] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2021-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
### Fixed
@ -376,7 +387,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12

View File

@ -25,39 +25,38 @@ configurations {
dependencies {
compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
implementation group: "commons-io", name: "commons-io", version: "2.7"
implementation group: "commons-io", name: "commons-io", version: "2.8.0"
implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.12.1")
implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.60.Final")
implementation group: "org.http4k", name: "http4k-core"
implementation group: "org.http4k", name: "http4k-resilience4j"
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
implementation group: "org.http4k", name: "http4k-format-jackson"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310"
implementation group: "org.http4k", name: "http4k-client-okhttp"
implementation group: "org.http4k", name: "http4k-metrics-micrometer"
implementation group: "org.http4k", name: "http4k-server-netty"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty", name: "netty-transport-native-epoll", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.36.Final"
implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'
implementation group: "com.h2database", name: "h2", version: "1.4.200"
implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0'
implementation group: "com.zaxxer", name: "HikariCP", version: "4.0.2"
implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.34.0"
implementation "org.ktorm:ktorm-core:$ktorm_version"
implementation "org.ktorm:ktorm-jackson:$ktorm_version"
implementation "info.picocli:picocli:4.5.0"
kapt "info.picocli:picocli-codegen:4.5.0"
implementation "info.picocli:picocli:$picocli_version"
kapt "info.picocli:picocli-codegen:$picocli_version"
testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
testImplementation "io.kotest:kotest-assertions-core:$kotest_version"

View File

@ -1,3 +1,4 @@
http_4k_version=4.3.0.0
kotest_version=4.4.1
ktorm_version=3.2.0
ktorm_version=3.3.0
picocli_version=4.6.1

View File

@ -30,6 +30,7 @@ public class CachingInputStream extends ProxyInputStream {
private final OutputStream cache;
private final ExecutorService executor;
private final Runnable onClose;
private boolean eofReached = false;
public CachingInputStream(InputStream response, ExecutorService executor, OutputStream cache, Runnable onClose) {
super(response);
@ -40,7 +41,7 @@ public class CachingInputStream extends ProxyInputStream {
@Override
public void close() throws IOException {
if (read() == EOF) {
if (eofReached) {
try {
in.close();
} catch (IOException ignored) {
@ -50,25 +51,24 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
}
onClose.run();
return;
} else {
executor.submit(() -> {
try {
IOUtils.copy(in, cache);
} catch (IOException ignored) {
} finally {
try {
in.close();
} catch (IOException ignored) {
}
try {
cache.close();
} catch (IOException ignored) {
}
onClose.run();
}
});
}
executor.submit(() -> {
try {
IOUtils.copy(in, cache);
} catch (IOException ignored) {
} finally {
try {
in.close();
} catch (IOException ignored) {
}
try {
cache.close();
} catch (IOException ignored) {
}
onClose.run();
}
});
}
@Override
@ -80,6 +80,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return ch;
}
@ -93,6 +95,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return n;
}
@ -106,6 +110,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return n;
}

View File

@ -23,15 +23,17 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info
import mdnet.settings.*
import org.http4k.core.Body
import org.http4k.core.HttpHandler
import org.http4k.core.Method
import org.http4k.core.Request
import okhttp3.Dns
import okhttp3.OkHttpClient
import org.http4k.client.OkHttp
import org.http4k.core.*
import org.http4k.format.ConfigurableJackson
import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings
import org.http4k.lens.LensFailure
import org.slf4j.LoggerFactory
import java.net.Inet4Address
import java.net.InetAddress
object ServerHandlerJackson : ConfigurableJackson(
KotlinModule()
@ -41,7 +43,16 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
)
class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
class BackendApi(private val settings: ClientSettings) {
private val client = OkHttp(
client = OkHttpClient.Builder()
.dns(object : Dns {
override fun lookup(hostname: String): List<InetAddress> {
return Dns.SYSTEM.lookup(hostname).filterIsInstance<Inet4Address>()
}
})
.build()
)
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
fun logoutFromControl(): Boolean {

View File

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 29
const val CLIENT_BUILD = 30
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

View File

@ -97,8 +97,6 @@ class Main : Runnable {
throw IllegalArgumentException("Cache folder $cacheFolder must be a directory")
}
migrate(databaseFolder)
val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder)
val hook = Thread {
client.shutdown()

View File

@ -198,7 +198,7 @@ class MangaDexClient(private val settingsFile: File, databaseFolder: Path, cache
}
private fun validateSettings(settings: ClientSettings) {
if (settings.maxCacheSizeInMebibytes < 20480) {
if (settings.maxCacheSizeInMebibytes < 40960) {
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 20480 MiB (20 GiB)")
}

View File

@ -1,75 +0,0 @@
/*
Mangadex@Home
Copyright (c) 2020, MangaDex Network
This file is part of MangaDex@Home.
MangaDex@Home is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MangaDex@Home is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet
import mdnet.cache.DbImage
import mdnet.cache.INIT_TABLE
import org.ktorm.database.Database
import org.ktorm.dsl.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
fun main() {
migrate(Paths.get("./"))
}
fun migrate(path: Path) {
val h2file = path.resolve("metadata.mv.db")
if (!Files.exists(h2file)) {
return
}
println("Migrating database - this may take a long time")
Class.forName("org.sqlite.JDBC")
val sqliteDb = path.resolve("metadata.db")
Files.deleteIfExists(sqliteDb)
val sqlite = Database.connect("jdbc:sqlite:$sqliteDb")
sqlite.useConnection { conn ->
conn.prepareStatement(INIT_TABLE).use {
it.execute()
}
}
val db = path.resolve("metadata")
val h2 = Database.connect("jdbc:h2:$db")
h2.useConnection { conn ->
conn.prepareStatement(INIT_TABLE).use {
it.execute()
}
}
h2.from(DbImage).select().asIterable().chunked(1000).forEach { list ->
sqlite.batchInsert(DbImage) {
for (data in list) {
item {
set(DbImage.id, data[DbImage.id])
set(DbImage.accessed, data[DbImage.accessed])
set(DbImage.size, data[DbImage.size])
}
}
}
}
Files.move(h2file, path.resolve("metadata.mv.db.old"))
}

View File

@ -20,7 +20,6 @@ package mdnet
import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import io.netty.util.internal.SystemPropertyUtil
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.logging.error
@ -30,6 +29,7 @@ import mdnet.metrics.DefaultMicrometerMetrics
import mdnet.server.getServer
import mdnet.settings.ClientSettings
import mdnet.settings.RemoteSettings
import okhttp3.ConnectionPool
import okhttp3.OkHttpClient
import okhttp3.Protocol
import org.http4k.client.OkHttp
@ -78,17 +78,17 @@ class ServerManager(
OkHttp(
bodyMode = BodyMode.Stream,
client = OkHttpClient.Builder()
.callTimeout(Duration.ofSeconds(30))
.connectTimeout(Duration.ofSeconds(1))
.writeTimeout(Duration.ofSeconds(5))
.readTimeout(Duration.ofSeconds(5))
.let {
if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
it.protocols(listOf(Protocol.HTTP_1_1))
} else {
it
}
}
.connectTimeout(Duration.ofSeconds(2))
.connectionPool(
ConnectionPool(
maxIdleConnections = 100,
keepAliveDuration = 1,
timeUnit = TimeUnit.MINUTES
)
)
.writeTimeout(Duration.ofSeconds(10))
.readTimeout(Duration.ofSeconds(10))
.protocols(listOf(Protocol.HTTP_1_1))
.build()
)
)
@ -100,7 +100,7 @@ class ServerManager(
init {
state = Uninitialized
backendApi = BackendApi(settings, OkHttp())
backendApi = BackendApi(settings)
}
fun start() {

View File

@ -64,6 +64,7 @@ import java.security.PrivateKey
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import java.util.Locale
import java.util.concurrent.TimeUnit
import javax.net.ssl.SSLException
sealed class NettyTransport(threads: Int) {
@ -76,12 +77,6 @@ sealed class NettyTransport(threads: Int) {
}
)
fun shutdownGracefully() {
bossGroup.shutdownGracefully().sync()
workerGroup.shutdownGracefully().sync()
executor.shutdownGracefully().sync()
}
private class NioTransport(threads: Int) : NettyTransport(threads) {
override val bossGroup = NioEventLoopGroup(1)
override val workerGroup = NioEventLoopGroup(8)
@ -108,7 +103,7 @@ sealed class NettyTransport(threads: Int) {
val name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim { it <= ' ' }
val threadsToUse = if (threads == 0) defaultNumThreads() else threads
LOGGER.info { "Choosing a transport using $threadsToUse threads" }
LOGGER.info { "Choosing a transport with $threadsToUse threads" }
if (name.startsWith("linux")) {
if (!SystemPropertyUtil.get("no-iouring").toBoolean()) {
@ -116,7 +111,7 @@ sealed class NettyTransport(threads: Int) {
LOGGER.info { "Using IOUring transport" }
return IOUringTransport(threadsToUse)
} else {
LOGGER.info(IOUring.unavailabilityCause()) {
LOGGER.info {
"IOUring transport not available (this may be normal)"
}
}
@ -127,7 +122,7 @@ sealed class NettyTransport(threads: Int) {
LOGGER.info { "Using Epoll transport" }
return EpollTransport(threadsToUse)
} else {
LOGGER.info(Epoll.unavailabilityCause()) {
LOGGER.info {
"Epoll transport not available (this may be normal)"
}
}
@ -145,7 +140,7 @@ class Netty(
private val serverSettings: ServerSettings,
private val statistics: Statistics
) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
private val transport = NettyTransport.bestForPlatform(serverSettings.threads)
private lateinit var channel: Channel
@ -190,7 +185,7 @@ class Netty(
)
ch.pipeline().addLast("streamer", ChunkedWriteHandler())
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(httpHandler))
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(http))
ch.pipeline().addLast(
"exceptions",
@ -216,10 +211,14 @@ class Netty(
override fun stop() = apply {
channel.close().sync()
transport.shutdownGracefully()
transport.run {
bossGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
workerGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
executor.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
}
}
override fun port(): Int = serverSettings.port
override fun port(): Int = (channel.localAddress() as InetSocketAddress).port
}
companion object {

View File

@ -105,9 +105,6 @@ fun getServer(
val verifier = TokenVerifier(
tokenKey = remoteSettings.tokenKey,
shouldVerify = { chapter, _ ->
!remoteSettings.disableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e")
}
)
return timeRequest()

View File

@ -37,56 +37,53 @@ import org.slf4j.LoggerFactory
import java.time.OffsetDateTime
import java.util.Base64
class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, String) -> Boolean) : Filter {
class TokenVerifier(tokenKey: ByteArray) : Filter {
private val box = TweetNaclFast.SecretBox(tokenKey)
override fun invoke(next: HttpHandler): HttpHandler {
return then@{
val chapterHash = Path.of("chapterHash")(it)
val fileName = Path.of("fileName")(it)
if (shouldVerify(chapterHash, fileName)) {
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
val tokenArr = try {
val toDecode = try {
Path.of("token")(it)
} catch (e: LensFailure) {
LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" }
return@then Response(Status.FORBIDDEN).body("Token is missing")
}
Base64.getUrlDecoder().decode(toDecode)
} catch (e: IllegalArgumentException) {
LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" }
return@then Response(Status.FORBIDDEN).body("Token is invalid base64")
val tokenArr = try {
val toDecode = try {
Path.of("token")(it)
} catch (e: LensFailure) {
LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" }
return@then Response(Status.FORBIDDEN).body("Token is missing")
}
if (tokenArr.size < 24) {
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN)
}
val token = try {
JACKSON.readValue<Token>(
box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply {
if (this == null) {
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN)
}
Base64.getUrlDecoder().decode(toDecode)
} catch (e: IllegalArgumentException) {
LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" }
return@then Response(Status.FORBIDDEN).body("Token is invalid base64")
}
if (tokenArr.size < 24) {
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN)
}
val token = try {
JACKSON.readValue<Token>(
box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply {
if (this == null) {
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN)
}
)
} catch (e: JsonProcessingException) {
LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN).body("Token is invalid")
}
}
)
} catch (e: JsonProcessingException) {
LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN).body("Token is invalid")
}
if (OffsetDateTime.now().isAfter(token.expires)) {
LOGGER.info { "Request for $cleanedUri rejected for expired token" }
return@then Response(Status.GONE).body("Token has expired")
}
if (OffsetDateTime.now().isAfter(token.expires)) {
LOGGER.info { "Request for $cleanedUri rejected for expired token" }
return@then Response(Status.GONE).body("Token has expired")
}
if (token.hash != chapterHash) {
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
}
if (token.hash != chapterHash) {
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
}
return@then next(it)

View File

@ -42,7 +42,7 @@ class ImageStorageTest : FreeSpec() {
val imageStorage = ImageStorage(
maxSize = 5,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
@ -160,7 +160,7 @@ class ImageStorageSlowTest : FreeSpec() {
val imageStorage = ImageStorage(
maxSize = 4097,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
)
"autoPrune" - {

View File

@ -112,7 +112,7 @@ class ImageServerTest : FreeSpec() {
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
@ -157,7 +157,7 @@ class ImageServerTest : FreeSpec() {
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)

View File

@ -31,9 +31,7 @@ class TokenVerifierTest : FreeSpec() {
val clientKeys = TweetNaclFast.Box.keyPair()
val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey)
val backend = TokenVerifier(box.before()) { _, _ ->
true
}.then {
val backend = TokenVerifier(box.before()).then {
Response(Status.OK)
}