From 22e318dafdd0508b95be7d211f0eda556fc057f8 Mon Sep 17 00:00:00 2001 From: carbotaniuman Date: Fri, 28 May 2021 19:06:55 +0000 Subject: [PATCH] Cut 2.0.1 --- CHANGELOG.md | 10 +++++ src/main/kotlin/mdnet/Constants.kt | 2 +- src/main/kotlin/mdnet/Main.kt | 22 +---------- src/main/kotlin/mdnet/ServerManager.kt | 1 + .../kotlin/mdnet/netty/ApplicationNetty.kt | 38 +++++++++++++++++-- src/main/kotlin/mdnet/server/ImageHandler.kt | 6 +-- src/main/kotlin/mdnet/server/ImageServer.kt | 8 ++-- src/main/kotlin/mdnet/server/common.kt | 11 ++++-- src/main/kotlin/mdnet/server/crypto.kt | 2 +- .../kotlin/mdnet/settings/ClientSettings.kt | 4 +- .../kotlin/mdnet/server/ImageServerTest.kt | 7 ++++ 11 files changed, 76 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a00c4ff..303cbfc 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [2.0.1] - 2021-05-27 +### Added +- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman]. + +### Changed +- [2021-05-21] Update metrics and fix cache directory leak [@carbotaniuman]. +- [2021-05-21] Change headers to be wildcards [@carbotaniuman]. +- [2021-05-27] Make sending the `Server` header configurable but off by default [@carbotaniuman]. + ## [2.0.0] - 2021-03-11 ### Changed - [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman]. @@ -388,6 +397,7 @@ This release contains many breaking changes! Of note are the changes to the cach - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. [Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD +[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1 [2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13 diff --git a/src/main/kotlin/mdnet/Constants.kt b/src/main/kotlin/mdnet/Constants.kt index 92587fc..e1a4c25 100644 --- a/src/main/kotlin/mdnet/Constants.kt +++ b/src/main/kotlin/mdnet/Constants.kt @@ -21,7 +21,7 @@ package mdnet import java.time.Duration object Constants { - const val CLIENT_BUILD = 30 + const val CLIENT_BUILD = 31 @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) diff --git a/src/main/kotlin/mdnet/Main.kt b/src/main/kotlin/mdnet/Main.kt index ee2b884..f15090e 100644 --- a/src/main/kotlin/mdnet/Main.kt +++ b/src/main/kotlin/mdnet/Main.kt @@ -67,26 +67,8 @@ class Main : Runnable { """.trimIndent() ) - if (!Files.isDirectory(databaseFolder) || Files.isRegularFile(databaseFolder.resolveSibling(databaseFolder.fileName.toString() + ".mv.db"))) { - println() - println() - println( - """the --database option now takes in the folder with the database file! - |(it previously took in the path to the file without any extensions) - |if you are using docker update your docker mount settings! - |if you are not, manually move update your --database args! - |note: the database file itself should be named metadata.{extension} - |where {extension} can be `.db` or `.mv.db` - | - |If this is your first time seeing this message, please check out the support - |channel as things HAVE changed. Failure to do so WILL require - |a cache wipe. - """.trimMargin() - ) - println() - println() - - throw IllegalArgumentException() + if (!Files.isDirectory(databaseFolder)) { + throw IllegalArgumentException("Expected $databaseFolder to be a folder, was a file") } if (Files.isRegularFile(databaseFolder)) { diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index 03f7427..b024523 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -264,6 +264,7 @@ class ServerManager( storage, remoteSettings, settings.serverSettings, + settings.devSettings, settings.metricsSettings, statistics, registry, diff --git a/src/main/kotlin/mdnet/netty/ApplicationNetty.kt b/src/main/kotlin/mdnet/netty/ApplicationNetty.kt index 73a6f4b..016084e 100644 --- a/src/main/kotlin/mdnet/netty/ApplicationNetty.kt +++ b/src/main/kotlin/mdnet/netty/ApplicationNetty.kt @@ -30,6 +30,8 @@ import io.netty.handler.codec.DecoderException import io.netty.handler.codec.http.HttpObjectAggregator import io.netty.handler.codec.http.HttpServerCodec import io.netty.handler.codec.http.HttpServerKeepAliveHandler +import io.netty.handler.ssl.SniCompletionEvent +import io.netty.handler.ssl.SniHandler import io.netty.handler.ssl.SslContextBuilder import io.netty.handler.stream.ChunkedWriteHandler import io.netty.handler.timeout.ReadTimeoutException @@ -41,6 +43,7 @@ import io.netty.handler.traffic.TrafficCounter import io.netty.incubator.channel.uring.IOUring import io.netty.incubator.channel.uring.IOUringEventLoopGroup import io.netty.incubator.channel.uring.IOUringServerSocketChannel +import io.netty.util.DomainWildcardMappingBuilder import io.netty.util.concurrent.DefaultEventExecutorGroup import io.netty.util.internal.SystemPropertyUtil import mdnet.Constants @@ -48,8 +51,9 @@ import mdnet.data.Statistics import mdnet.logging.info import mdnet.logging.trace import mdnet.logging.warn +import mdnet.settings.DevSettings +import mdnet.settings.RemoteSettings import mdnet.settings.ServerSettings -import mdnet.settings.TlsCert import org.http4k.core.HttpHandler import org.http4k.server.Http4kChannelHandler import org.http4k.server.Http4kServer @@ -136,8 +140,9 @@ sealed class NettyTransport(threads: Int) { } class Netty( - private val tls: TlsCert, + private val remoteSettings: RemoteSettings, private val serverSettings: ServerSettings, + private val devSettings: DevSettings, private val statistics: Statistics ) : ServerConfig { override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer { @@ -155,6 +160,7 @@ class Netty( override fun start(): Http4kServer = apply { LOGGER.info { "Starting Netty!" } + val tls = remoteSettings.tls!! val certs = getX509Certs(tls.certificate) val sslContext = SslContextBuilder @@ -167,7 +173,33 @@ class Netty( .channelFactory(transport.factory) .childHandler(object : ChannelInitializer() { public override fun initChannel(ch: SocketChannel) { - ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc())) + ch.pipeline().addLast( + "ssl", + SniHandler(DomainWildcardMappingBuilder(sslContext).build()) + ) + + ch.pipeline().addLast( + "dropHostname", + object : ChannelInboundHandlerAdapter() { + private val hostToTest = remoteSettings.url.authority.let { + it.substring(0, it.lastIndexOf(":")) + } + + override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { + if (evt is SniCompletionEvent) { + if (!devSettings.disableSniCheck) { + if (!evt.hostname().endsWith(hostToTest) && + !evt.hostname().endsWith("localhost") + ) { + ctx.close() + } + } + } else { + ctx.fireUserEventTriggered(evt) + } + } + } + ) ch.pipeline().addLast("codec", HttpServerCodec()) ch.pipeline().addLast("keepAlive", HttpServerKeepAliveHandler()) diff --git a/src/main/kotlin/mdnet/server/ImageHandler.kt b/src/main/kotlin/mdnet/server/ImageHandler.kt index 5c922f4..de6a785 100644 --- a/src/main/kotlin/mdnet/server/ImageHandler.kt +++ b/src/main/kotlin/mdnet/server/ImageHandler.kt @@ -147,7 +147,7 @@ class ImageServer( } respondWithImage(tee, contentLength, contentType, lastModified, false) } else { - LOGGER.info { "Request for $sanitizedUri is being served due to write errors" } + LOGGER.info { "Request for $sanitizedUri is being served as the cache is full" } respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) } } @@ -181,10 +181,10 @@ class ImageServer( .then { next: HttpHandler -> { request: Request -> val response = next(request) - response.header("access-control-allow-origin", "https://mangadex.org") + response.header("access-control-allow-origin", "*") .header("access-control-expose-headers", "*") .header("access-control-allow-methods", "GET") - .header("timing-allow-origin", "https://mangadex.org") + .header("timing-allow-origin", "*") } } } diff --git a/src/main/kotlin/mdnet/server/ImageServer.kt b/src/main/kotlin/mdnet/server/ImageServer.kt index 1af0f1a..12532a8 100644 --- a/src/main/kotlin/mdnet/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/server/ImageServer.kt @@ -31,6 +31,7 @@ import mdnet.logging.warn import mdnet.metrics.GeoIpMetricsFilterBuilder import mdnet.metrics.PostTransactionLabeler import mdnet.netty.Netty +import mdnet.settings.DevSettings import mdnet.settings.MetricsSettings import mdnet.settings.RemoteSettings import mdnet.settings.ServerSettings @@ -46,6 +47,7 @@ fun getServer( storage: ImageStorage, remoteSettings: RemoteSettings, serverSettings: ServerSettings, + devSettings: DevSettings, metricsSettings: MetricsSettings, statistics: Statistics, registry: PrometheusMeterRegistry, @@ -75,7 +77,7 @@ fun getServer( } circuitBreaker.eventPublisher.onReset { - LOGGER.warn { "Circuit breaker has rest" } + LOGGER.warn { "Circuit breaker has reset" } } circuitBreaker.eventPublisher.onStateTransition { @@ -109,7 +111,7 @@ fun getServer( ) return timeRequest() - .then(addCommonHeaders()) + .then(addCommonHeaders(devSettings.sendServerHeader)) .then(catchAllHideDetails()) .then( routes( @@ -142,7 +144,7 @@ fun getServer( GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build() ) ) - .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) + .asServer(Netty(remoteSettings, serverSettings, devSettings, statistics)) } private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) diff --git a/src/main/kotlin/mdnet/server/common.kt b/src/main/kotlin/mdnet/server/common.kt index aeae7fd..9cd4e72 100644 --- a/src/main/kotlin/mdnet/server/common.kt +++ b/src/main/kotlin/mdnet/server/common.kt @@ -35,12 +35,17 @@ import java.util.* private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH) private val LOGGER = LoggerFactory.getLogger("Application") -fun addCommonHeaders(): Filter { +fun addCommonHeaders(sendServerHeader: Boolean): Filter { return Filter { next: HttpHandler -> { request: Request -> val response = next(request) - response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))) - .header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})") + response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))).let { + if (sendServerHeader) { + it.header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})") + } else { + it + } + } } } } diff --git a/src/main/kotlin/mdnet/server/crypto.kt b/src/main/kotlin/mdnet/server/crypto.kt index 04bb9f4..9f8e5ec 100644 --- a/src/main/kotlin/mdnet/server/crypto.kt +++ b/src/main/kotlin/mdnet/server/crypto.kt @@ -20,7 +20,7 @@ package mdnet.server import java.security.MessageDigest -fun md5Bytes(stringToHash: String) = +fun md5Bytes(stringToHash: String): ByteArray = MessageDigest.getInstance("MD5") .digest(stringToHash.toByteArray()) diff --git a/src/main/kotlin/mdnet/settings/ClientSettings.kt b/src/main/kotlin/mdnet/settings/ClientSettings.kt index 5f416e6..e8a52d5 100644 --- a/src/main/kotlin/mdnet/settings/ClientSettings.kt +++ b/src/main/kotlin/mdnet/settings/ClientSettings.kt @@ -47,7 +47,9 @@ data class ServerSettings( @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) data class DevSettings( - val devUrl: String? = null + val devUrl: String? = null, + val disableSniCheck: Boolean = false, + val sendServerHeader: Boolean = false, ) @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) diff --git a/src/test/kotlin/mdnet/server/ImageServerTest.kt b/src/test/kotlin/mdnet/server/ImageServerTest.kt index 10cf4a7..60567ba 100644 --- a/src/test/kotlin/mdnet/server/ImageServerTest.kt +++ b/src/test/kotlin/mdnet/server/ImageServerTest.kt @@ -39,6 +39,7 @@ import org.http4k.core.Response import org.http4k.core.Status import org.http4k.kotest.shouldHaveHeader import org.http4k.kotest.shouldHaveStatus +import org.http4k.kotest.shouldNotHaveHeader import org.http4k.routing.bind import org.http4k.routing.routes import org.ktorm.database.Database @@ -106,6 +107,12 @@ class ImageServerTest : FreeSpec() { response.close() } } + + "should not have Server header" { + val response = handler(Request(Method.GET, "/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) + response.shouldNotHaveHeader("Server") + response.close() + } } "with real cache" - {