1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Use HTTP/2

This commit is contained in:
carbotaniuman 2021-03-02 12:22:24 -06:00
parent 2252a9f02d
commit 1be9ee8506
8 changed files with 71 additions and 83 deletions

View file

@ -8,19 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ### Added
### Changed ### Changed
- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9]
- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9]
- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9]
### Deprecated ### Deprecated
### Removed ### Removed
### Fixed ### Fixed
- [2020-02-21] Fix pipeline [@_tde9]
### Security ### Security
## [2.0.0-rc14] - 2021-03-02
### Changed
- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2020-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
### Fixed
- [2020-02-21] Fix pipeline [@_tde9].
## [2.0.0-rc13] - 2021-02-19 ## [2.0.0-rc13] - 2021-02-19
### Changed ### Changed
- [2021-02-19] Back to sqlite we go [@carbotaniuman]. - [2021-02-19] Back to sqlite we go [@carbotaniuman].
@ -369,7 +375,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed ### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...HEAD [Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12 [2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12
[2.0.0-rc11]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...2.0.0-rc11 [2.0.0-rc11]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...2.0.0-rc11

View file

@ -1,7 +1,7 @@
plugins { plugins {
id "jacoco" id "jacoco"
id "java" id "java"
id "org.jetbrains.kotlin.jvm" version "1.4.20" id "org.jetbrains.kotlin.jvm" version "1.4.30"
id "org.jetbrains.kotlin.kapt" version "1.4.0" id "org.jetbrains.kotlin.kapt" version "1.4.0"
id "application" id "application"
id "com.github.johnrengelman.shadow" version "5.2.0" id "com.github.johnrengelman.shadow" version "5.2.0"
@ -32,18 +32,22 @@ dependencies {
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2" implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0" implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
implementation group: "org.http4k", name: "http4k-resilience4j", version: "$http_4k_version"
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
implementation group: "org.http4k", name: "http4k-core"
implementation group: "org.http4k", name: "http4k-resilience4j"
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1" implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-format-jackson"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-okhttp"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64" implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version" testImplementation group: "org.http4k", name: "http4k-testing-kotest"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final" runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1' implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'

View file

@ -1,4 +1,3 @@
http_4k_version=4.1.0.0 http_4k_version=4.3.0.0
exposed_version=0.26.2 kotest_version=4.4.1
kotest_version=4.4.0.RC1
ktorm_version=3.2.0 ktorm_version=3.2.0

View file

@ -23,7 +23,6 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info import mdnet.logging.info
import mdnet.settings.* import mdnet.settings.*
import org.http4k.client.ApacheClient
import org.http4k.core.Body import org.http4k.core.Body
import org.http4k.core.HttpHandler import org.http4k.core.HttpHandler
import org.http4k.core.Method import org.http4k.core.Method
@ -42,9 +41,8 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
) )
class BackendApi(private val settings: ClientSettings) { class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
private val client = ApacheClient()
fun logoutFromControl(): Boolean { fun logoutFromControl(): Boolean {
val serverSettings = settings.serverSettings val serverSettings = settings.serverSettings

View file

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration import java.time.Duration
object Constants { object Constants {
const val CLIENT_BUILD = 28 const val CLIENT_BUILD = 29
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

View file

@ -77,6 +77,10 @@ class Main : Runnable {
|if you are not, manually move update your --database args! |if you are not, manually move update your --database args!
|note: the database file itself should be named metadata.{extension} |note: the database file itself should be named metadata.{extension}
|where {extension} can be `.db` or `.mv.db` |where {extension} can be `.db` or `.mv.db`
|
|If this is your first time seeing this message, please check out the support
|channel as things HAVE changed. Failure to do so WILL require
|a cache wipe.
""".trimMargin() """.trimMargin()
) )
println() println()

View file

@ -20,6 +20,7 @@ package mdnet
import io.micrometer.prometheus.PrometheusConfig import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import io.netty.util.internal.SystemPropertyUtil
import mdnet.cache.ImageStorage import mdnet.cache.ImageStorage
import mdnet.data.Statistics import mdnet.data.Statistics
import mdnet.logging.error import mdnet.logging.error
@ -29,19 +30,16 @@ import mdnet.metrics.DefaultMicrometerMetrics
import mdnet.server.getServer import mdnet.server.getServer
import mdnet.settings.ClientSettings import mdnet.settings.ClientSettings
import mdnet.settings.RemoteSettings import mdnet.settings.RemoteSettings
import org.apache.hc.client5.http.config.RequestConfig import okhttp3.OkHttpClient
import org.apache.hc.client5.http.cookie.StandardCookieSpec import okhttp3.Protocol
import org.apache.hc.client5.http.impl.classic.HttpClients import org.http4k.client.OkHttp
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
import org.http4k.client.ApacheClient
import org.http4k.core.BodyMode import org.http4k.core.BodyMode
import org.http4k.core.then import org.http4k.core.then
import org.http4k.filter.ClientFilters import org.http4k.filter.ClientFilters
import org.http4k.filter.MicrometerMetrics import org.http4k.filter.MicrometerMetrics
import org.http4k.server.Http4kServer import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.time.Duration
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@ -73,26 +71,24 @@ class ServerManager(
private val executor = Executors.newSingleThreadScheduledExecutor() private val executor = Executors.newSingleThreadScheduledExecutor()
private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
private val statistics = Statistics() private val statistics = Statistics()
private val connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setMaxConnTotal(500) private val okhttp = ClientFilters.MicrometerMetrics.RequestCounter(registry)
.setMaxConnPerRoute(500)
.build()
private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry)
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
.then( .then(
ApacheClient( OkHttp(
responseBodyMode = BodyMode.Stream, bodyMode = BodyMode.Stream,
client = HttpClients.custom() client = OkHttpClient.Builder()
.disableConnectionState() .callTimeout(Duration.ofSeconds(30))
.setDefaultRequestConfig( .connectTimeout(Duration.ofSeconds(1))
RequestConfig.custom() .writeTimeout(Duration.ofSeconds(5))
.setCookieSpec(StandardCookieSpec.IGNORE) .readTimeout(Duration.ofSeconds(5))
.setConnectTimeout(Timeout.ofSeconds(2)) .let {
.setResponseTimeout(Timeout.ofSeconds(2)) if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
.setConnectionRequestTimeout(Timeout.ofSeconds(1)) it.protocols(listOf(Protocol.HTTP_1_1))
.build() } else {
) it
.setConnectionManager(connectionManager) }
}
.build() .build()
) )
) )
@ -104,7 +100,7 @@ class ServerManager(
init { init {
state = Uninitialized state = Uninitialized
backendApi = BackendApi(settings) backendApi = BackendApi(settings, OkHttp())
} }
fun start() { fun start() {
@ -205,20 +201,6 @@ class ServerManager(
45, 45, TimeUnit.SECONDS 45, 45, TimeUnit.SECONDS
) )
executor.scheduleWithFixedDelay(
{
try {
LOGGER.info { "Closing old Apache HTTP connections" }
connectionManager.closeExpired()
connectionManager.closeIdle(TimeValue.ofSeconds(30))
} catch (e: Exception) {
LOGGER.warn(e) { "Old Apache HTTP connection closer failed" }
}
},
45, 45, TimeUnit.SECONDS
)
LOGGER.info { "Image server has started" } LOGGER.info { "Image server has started" }
} }
@ -271,7 +253,7 @@ class ServerManager(
settings.metricsSettings, settings.metricsSettings,
statistics, statistics,
registry, registry,
apache, okhttp,
).start() ).start()
this.state = Running(server, remoteSettings) this.state = Running(server, remoteSettings)

View file

@ -25,8 +25,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.logging.debug import mdnet.logging.debug
import mdnet.logging.warn import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils import org.http4k.client.OkHttp
import org.http4k.client.ApacheClient
import org.http4k.core.Filter import org.http4k.core.Filter
import org.http4k.core.HttpHandler import org.http4k.core.HttpHandler
import org.http4k.core.Method import org.http4k.core.Method
@ -37,7 +36,6 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.net.InetAddress import java.net.InetAddress
import java.net.UnknownHostException import java.net.UnknownHostException
import java.nio.file.Files
class GeoIpMetricsFilter( class GeoIpMetricsFilter(
private val databaseReader: DatabaseReader?, private val databaseReader: DatabaseReader?,
@ -92,7 +90,8 @@ class GeoIpMetricsFilterBuilder(
private val license: String, private val license: String,
private val registry: PrometheusMeterRegistry, private val registry: PrometheusMeterRegistry,
) { ) {
val client = ApacheClient() private val client = OkHttp()
fun build(): GeoIpMetricsFilter { fun build(): GeoIpMetricsFilter {
return if (enableGeoIp) { return if (enableGeoIp) {
LOGGER.info("GeoIp initialising") LOGGER.info("GeoIp initialising")
@ -105,32 +104,27 @@ class GeoIpMetricsFilterBuilder(
} }
private fun initDatabase(): DatabaseReader { private fun initDatabase(): DatabaseReader {
val databaseFileDir = Files.createTempDirectory("mangadex-geoip")
val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
val response = client(Request(Method.GET, geoIpDatabaseUri)) val response = client(Request(Method.GET, geoIpDatabaseUri))
if (response.status != Status.OK) { if (response.status != Status.OK) {
throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")
} }
response.use { return response.use { data ->
val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream) TarArchiveInputStream(data.body.gunzippedStream().stream).use {
var entry = archiveStream.nextTarEntry var entry = it.nextTarEntry
while (!entry.name.endsWith(".mmdb")) { while (!entry.name.endsWith(".mmdb")) {
LOGGER.debug { "Skipped non-database file: ${entry.name}" } LOGGER.debug { "Skipped non-database file: ${entry.name}" }
entry = archiveStream.nextTarEntry entry = it.nextTarEntry
}
// reads only the current entry to its end
DatabaseReader
.Builder(it)
.withCache(CHMCache())
.build()
} }
// reads only the current entry to its end
val dbBytes = IOUtils.toByteArray(archiveStream)
Files.write(databaseFile, dbBytes)
} }
return DatabaseReader
.Builder(databaseFile.toFile())
.withCache(CHMCache())
.build()
} }
companion object { companion object {