From 1be9ee850607e0f0885404812b2a6db114a7e3a1 Mon Sep 17 00:00:00 2001 From: carbotaniuman <41451839+carbotaniuman@users.noreply.github.com> Date: Tue, 2 Mar 2021 12:22:24 -0600 Subject: [PATCH] Use HTTP/2 --- CHANGELOG.md | 17 +++-- build.gradle | 20 +++--- gradle.properties | 5 +- src/main/kotlin/mdnet/BackendApi.kt | 4 +- src/main/kotlin/mdnet/Constants.kt | 2 +- src/main/kotlin/mdnet/Main.kt | 4 ++ src/main/kotlin/mdnet/ServerManager.kt | 64 +++++++------------ .../mdnet/metrics/GeoIpMetricsFilter.kt | 38 +++++------ 8 files changed, 71 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c7645..1c7e17d 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,19 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed -- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9] -- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9] -- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9] ### Deprecated ### Removed ### Fixed -- [2020-02-21] Fix pipeline [@_tde9] ### Security +## [2.0.0-rc14] - 2021-03-02 +### Changed +- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9]. +- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9]. +- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9]. +- [2020-02-25] Use HTTP/2 to download when possible [@carbotaniuman]. + +### Fixed +- [2020-02-21] Fix pipeline [@_tde9]. + ## [2.0.0-rc13] - 2021-02-19 ### Changed - [2021-02-19] Back to sqlite we go [@carbotaniuman]. @@ -369,7 +375,8 @@ This release contains many breaking changes! Of note are the changes to the cach ### Fixed - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. -[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...HEAD +[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD +[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13 [2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12 [2.0.0-rc11]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...2.0.0-rc11 diff --git a/build.gradle b/build.gradle index a83fc08..064fa78 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ plugins { id "jacoco" id "java" - id "org.jetbrains.kotlin.jvm" version "1.4.20" + id "org.jetbrains.kotlin.jvm" version "1.4.30" id "org.jetbrains.kotlin.kapt" version "1.4.0" id "application" id "com.github.johnrengelman.shadow" version "5.2.0" @@ -32,18 +32,22 @@ dependencies { implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2" implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0" - implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" - implementation group: "org.http4k", name: "http4k-resilience4j", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4" + + implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4") + + implementation group: "org.http4k", name: "http4k-core" + implementation group: "org.http4k", name: "http4k-resilience4j" implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1" - implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-format-jackson" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" - implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" - implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" - implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-client-okhttp" + implementation group: "org.http4k", name: "http4k-metrics-micrometer" + implementation group: "org.http4k", name: "http4k-server-netty" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64" - testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version" + testImplementation group: "org.http4k", name: "http4k-testing-kotest" runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final" implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1' diff --git a/gradle.properties b/gradle.properties index 46d7dbc..a267748 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,3 @@ -http_4k_version=4.1.0.0 -exposed_version=0.26.2 -kotest_version=4.4.0.RC1 +http_4k_version=4.3.0.0 +kotest_version=4.4.1 ktorm_version=3.2.0 \ No newline at end of file diff --git a/src/main/kotlin/mdnet/BackendApi.kt b/src/main/kotlin/mdnet/BackendApi.kt index ba4e9c7..9d8cbb0 100644 --- a/src/main/kotlin/mdnet/BackendApi.kt +++ b/src/main/kotlin/mdnet/BackendApi.kt @@ -23,7 +23,6 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import mdnet.ServerHandlerJackson.auto import mdnet.logging.info import mdnet.settings.* -import org.http4k.client.ApacheClient import org.http4k.core.Body import org.http4k.core.HttpHandler import org.http4k.core.Method @@ -42,9 +41,8 @@ object ServerHandlerJackson : ConfigurableJackson( .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) ) -class BackendApi(private val settings: ClientSettings) { +class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) { private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS - private val client = ApacheClient() fun logoutFromControl(): Boolean { val serverSettings = settings.serverSettings diff --git a/src/main/kotlin/mdnet/Constants.kt b/src/main/kotlin/mdnet/Constants.kt index 1d5d7b4..06c8e76 100644 --- a/src/main/kotlin/mdnet/Constants.kt +++ b/src/main/kotlin/mdnet/Constants.kt @@ -21,7 +21,7 @@ package mdnet import java.time.Duration object Constants { - const val CLIENT_BUILD = 28 + const val CLIENT_BUILD = 29 @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) diff --git a/src/main/kotlin/mdnet/Main.kt b/src/main/kotlin/mdnet/Main.kt index e39f2ee..369b2d0 100644 --- a/src/main/kotlin/mdnet/Main.kt +++ b/src/main/kotlin/mdnet/Main.kt @@ -77,6 +77,10 @@ class Main : Runnable { |if you are not, manually move update your --database args! |note: the database file itself should be named metadata.{extension} |where {extension} can be `.db` or `.mv.db` + | + |If this is your first time seeing this message, please check out the support + |channel as things HAVE changed. Failure to do so WILL require + |a cache wipe. """.trimMargin() ) println() diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index c3e4fe8..2d820ed 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -20,6 +20,7 @@ package mdnet import io.micrometer.prometheus.PrometheusConfig import io.micrometer.prometheus.PrometheusMeterRegistry +import io.netty.util.internal.SystemPropertyUtil import mdnet.cache.ImageStorage import mdnet.data.Statistics import mdnet.logging.error @@ -29,19 +30,16 @@ import mdnet.metrics.DefaultMicrometerMetrics import mdnet.server.getServer import mdnet.settings.ClientSettings import mdnet.settings.RemoteSettings -import org.apache.hc.client5.http.config.RequestConfig -import org.apache.hc.client5.http.cookie.StandardCookieSpec -import org.apache.hc.client5.http.impl.classic.HttpClients -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder -import org.apache.hc.core5.util.TimeValue -import org.apache.hc.core5.util.Timeout -import org.http4k.client.ApacheClient +import okhttp3.OkHttpClient +import okhttp3.Protocol +import org.http4k.client.OkHttp import org.http4k.core.BodyMode import org.http4k.core.then import org.http4k.filter.ClientFilters import org.http4k.filter.MicrometerMetrics import org.http4k.server.Http4kServer import org.slf4j.LoggerFactory +import java.time.Duration import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -73,26 +71,24 @@ class ServerManager( private val executor = Executors.newSingleThreadScheduledExecutor() private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) private val statistics = Statistics() - private val connectionManager = PoolingHttpClientConnectionManagerBuilder.create() - .setMaxConnTotal(500) - .setMaxConnPerRoute(500) - .build() - private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry) + + private val okhttp = ClientFilters.MicrometerMetrics.RequestCounter(registry) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) .then( - ApacheClient( - responseBodyMode = BodyMode.Stream, - client = HttpClients.custom() - .disableConnectionState() - .setDefaultRequestConfig( - RequestConfig.custom() - .setCookieSpec(StandardCookieSpec.IGNORE) - .setConnectTimeout(Timeout.ofSeconds(2)) - .setResponseTimeout(Timeout.ofSeconds(2)) - .setConnectionRequestTimeout(Timeout.ofSeconds(1)) - .build() - ) - .setConnectionManager(connectionManager) + OkHttp( + bodyMode = BodyMode.Stream, + client = OkHttpClient.Builder() + .callTimeout(Duration.ofSeconds(30)) + .connectTimeout(Duration.ofSeconds(1)) + .writeTimeout(Duration.ofSeconds(5)) + .readTimeout(Duration.ofSeconds(5)) + .let { + if (SystemPropertyUtil.get("no-client-http2").toBoolean()) { + it.protocols(listOf(Protocol.HTTP_1_1)) + } else { + it + } + } .build() ) ) @@ -104,7 +100,7 @@ class ServerManager( init { state = Uninitialized - backendApi = BackendApi(settings) + backendApi = BackendApi(settings, OkHttp()) } fun start() { @@ -205,20 +201,6 @@ class ServerManager( 45, 45, TimeUnit.SECONDS ) - executor.scheduleWithFixedDelay( - { - try { - LOGGER.info { "Closing old Apache HTTP connections" } - - connectionManager.closeExpired() - connectionManager.closeIdle(TimeValue.ofSeconds(30)) - } catch (e: Exception) { - LOGGER.warn(e) { "Old Apache HTTP connection closer failed" } - } - }, - 45, 45, TimeUnit.SECONDS - ) - LOGGER.info { "Image server has started" } } @@ -271,7 +253,7 @@ class ServerManager( settings.metricsSettings, statistics, registry, - apache, + okhttp, ).start() this.state = Running(server, remoteSettings) diff --git a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt index a69ff2d..a1f29fd 100644 --- a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt +++ b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt @@ -25,8 +25,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry import mdnet.logging.debug import mdnet.logging.warn import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.io.IOUtils -import org.http4k.client.ApacheClient +import org.http4k.client.OkHttp import org.http4k.core.Filter import org.http4k.core.HttpHandler import org.http4k.core.Method @@ -37,7 +36,6 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.net.InetAddress import java.net.UnknownHostException -import java.nio.file.Files class GeoIpMetricsFilter( private val databaseReader: DatabaseReader?, @@ -92,7 +90,8 @@ class GeoIpMetricsFilterBuilder( private val license: String, private val registry: PrometheusMeterRegistry, ) { - val client = ApacheClient() + private val client = OkHttp() + fun build(): GeoIpMetricsFilter { return if (enableGeoIp) { LOGGER.info("GeoIp initialising") @@ -105,32 +104,27 @@ class GeoIpMetricsFilterBuilder( } private fun initDatabase(): DatabaseReader { - val databaseFileDir = Files.createTempDirectory("mangadex-geoip") - val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb") - val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) val response = client(Request(Method.GET, geoIpDatabaseUri)) if (response.status != Status.OK) { throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") } - response.use { - val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream) - var entry = archiveStream.nextTarEntry - while (!entry.name.endsWith(".mmdb")) { - LOGGER.debug { "Skipped non-database file: ${entry.name}" } - entry = archiveStream.nextTarEntry + return response.use { data -> + TarArchiveInputStream(data.body.gunzippedStream().stream).use { + var entry = it.nextTarEntry + while (!entry.name.endsWith(".mmdb")) { + LOGGER.debug { "Skipped non-database file: ${entry.name}" } + entry = it.nextTarEntry + } + + // reads only the current entry to its end + DatabaseReader + .Builder(it) + .withCache(CHMCache()) + .build() } - - // reads only the current entry to its end - val dbBytes = IOUtils.toByteArray(archiveStream) - Files.write(databaseFile, dbBytes) } - - return DatabaseReader - .Builder(databaseFile.toFile()) - .withCache(CHMCache()) - .build() } companion object {