gitlab-com-mangadex-pub
/
mangadex_at_home
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git
1
0
Fork 1
Code Releases Activity

Cut 2.0.1

This commit is contained in:
carbotaniuman 2021-05-28 19:06:55 +00:00
parent 5b154a6b20
commit 22e318dafd
11 changed files with 76 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -17,6 +17,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security
## [2.0.1] - 2021-05-27
### Added
- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
### Changed
- [2021-05-21] Update metrics and fix cache directory leak [@carbotaniuman].
- [2021-05-21] Change headers to be wildcards [@carbotaniuman].
- [2021-05-27] Make sending the `Server` header configurable but off by default [@carbotaniuman].
## [2.0.0] - 2021-03-11
### Changed
- [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
@ -388,6 +397,7 @@ This release contains many breaking changes! Of note are the changes to the cach
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13

2
src/main/kotlin/mdnet/Constants.kt
View File

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 30
const val CLIENT_BUILD = 31
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

22
src/main/kotlin/mdnet/Main.kt
View File

@ -67,26 +67,8 @@ class Main : Runnable {
""".trimIndent()
)
if (!Files.isDirectory(databaseFolder) || Files.isRegularFile(databaseFolder.resolveSibling(databaseFolder.fileName.toString() + ".mv.db"))) {
println()
println()
println(
"""the --database option now takes in the folder with the database file!
|(it previously took in the path to the file without any extensions)
|if you are using docker update your docker mount settings!
|if you are not, manually move update your --database args!
|note: the database file itself should be named metadata.{extension}
|where {extension} can be `.db` or `.mv.db`
|
|If this is your first time seeing this message, please check out the support
|channel as things HAVE changed. Failure to do so WILL require
|a cache wipe.
""".trimMargin()
)
println()
println()
throw IllegalArgumentException()
if (!Files.isDirectory(databaseFolder)) {
throw IllegalArgumentException("Expected $databaseFolder to be a folder, was a file")
}
if (Files.isRegularFile(databaseFolder)) {

1
src/main/kotlin/mdnet/ServerManager.kt
View File

@ -264,6 +264,7 @@ class ServerManager(
storage,
remoteSettings,
settings.serverSettings,
settings.devSettings,
settings.metricsSettings,
statistics,
registry,

38
src/main/kotlin/mdnet/netty/ApplicationNetty.kt
View File

@ -30,6 +30,8 @@ import io.netty.handler.codec.DecoderException
import io.netty.handler.codec.http.HttpObjectAggregator
import io.netty.handler.codec.http.HttpServerCodec
import io.netty.handler.codec.http.HttpServerKeepAliveHandler
import io.netty.handler.ssl.SniCompletionEvent
import io.netty.handler.ssl.SniHandler
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.stream.ChunkedWriteHandler
import io.netty.handler.timeout.ReadTimeoutException
@ -41,6 +43,7 @@ import io.netty.handler.traffic.TrafficCounter
import io.netty.incubator.channel.uring.IOUring
import io.netty.incubator.channel.uring.IOUringEventLoopGroup
import io.netty.incubator.channel.uring.IOUringServerSocketChannel
import io.netty.util.DomainWildcardMappingBuilder
import io.netty.util.concurrent.DefaultEventExecutorGroup
import io.netty.util.internal.SystemPropertyUtil
import mdnet.Constants
@ -48,8 +51,9 @@ import mdnet.data.Statistics
import mdnet.logging.info
import mdnet.logging.trace
import mdnet.logging.warn
import mdnet.settings.DevSettings
import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings
import mdnet.settings.TlsCert
import org.http4k.core.HttpHandler
import org.http4k.server.Http4kChannelHandler
import org.http4k.server.Http4kServer
@ -136,8 +140,9 @@ sealed class NettyTransport(threads: Int) {
}
class Netty(
private val tls: TlsCert,
private val remoteSettings: RemoteSettings,
private val serverSettings: ServerSettings,
private val devSettings: DevSettings,
private val statistics: Statistics
) : ServerConfig {
override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
@ -155,6 +160,7 @@ class Netty(
override fun start(): Http4kServer = apply {
LOGGER.info { "Starting Netty!" }
val tls = remoteSettings.tls!!
val certs = getX509Certs(tls.certificate)
val sslContext = SslContextBuilder
@ -167,7 +173,33 @@ class Netty(
.channelFactory(transport.factory)
.childHandler(object : ChannelInitializer<SocketChannel>() {
public override fun initChannel(ch: SocketChannel) {
ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc()))
ch.pipeline().addLast(
"ssl",
SniHandler(DomainWildcardMappingBuilder(sslContext).build())
)
ch.pipeline().addLast(
"dropHostname",
object : ChannelInboundHandlerAdapter() {
private val hostToTest = remoteSettings.url.authority.let {
it.substring(0, it.lastIndexOf(":"))
}
override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
if (evt is SniCompletionEvent) {
if (!devSettings.disableSniCheck) {
if (!evt.hostname().endsWith(hostToTest) &&
!evt.hostname().endsWith("localhost")
) {
ctx.close()
}
}
} else {
ctx.fireUserEventTriggered(evt)
}
}
}
)
ch.pipeline().addLast("codec", HttpServerCodec())
ch.pipeline().addLast("keepAlive", HttpServerKeepAliveHandler())

6
src/main/kotlin/mdnet/server/ImageHandler.kt
View File

@ -147,7 +147,7 @@ class ImageServer(
}
respondWithImage(tee, contentLength, contentType, lastModified, false)
} else {
LOGGER.info { "Request for $sanitizedUri is being served due to write errors" }
LOGGER.info { "Request for $sanitizedUri is being served as the cache is full" }
respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
}
}
@ -181,10 +181,10 @@ class ImageServer(
.then { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("access-control-allow-origin", "https://mangadex.org")
response.header("access-control-allow-origin", "*")
.header("access-control-expose-headers", "*")
.header("access-control-allow-methods", "GET")
.header("timing-allow-origin", "https://mangadex.org")
.header("timing-allow-origin", "*")
}
}
}

8
src/main/kotlin/mdnet/server/ImageServer.kt
View File

@ -31,6 +31,7 @@ import mdnet.logging.warn
import mdnet.metrics.GeoIpMetricsFilterBuilder
import mdnet.metrics.PostTransactionLabeler
import mdnet.netty.Netty
import mdnet.settings.DevSettings
import mdnet.settings.MetricsSettings
import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings
@ -46,6 +47,7 @@ fun getServer(
storage: ImageStorage,
remoteSettings: RemoteSettings,
serverSettings: ServerSettings,
devSettings: DevSettings,
metricsSettings: MetricsSettings,
statistics: Statistics,
registry: PrometheusMeterRegistry,
@ -75,7 +77,7 @@ fun getServer(
}
circuitBreaker.eventPublisher.onReset {
LOGGER.warn { "Circuit breaker has rest" }
LOGGER.warn { "Circuit breaker has reset" }
}
circuitBreaker.eventPublisher.onStateTransition {
@ -109,7 +111,7 @@ fun getServer(
)
return timeRequest()
.then(addCommonHeaders())
.then(addCommonHeaders(devSettings.sendServerHeader))
.then(catchAllHideDetails())
.then(
routes(
@ -142,7 +144,7 @@ fun getServer(
GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build()
)
)
.asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))
.asServer(Netty(remoteSettings, serverSettings, devSettings, statistics))
}
private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)

11
src/main/kotlin/mdnet/server/common.kt
View File

@ -35,12 +35,17 @@ import java.util.*
private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
private val LOGGER = LoggerFactory.getLogger("Application")
fun addCommonHeaders(): Filter {
fun addCommonHeaders(sendServerHeader: Boolean): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})")
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))).let {
if (sendServerHeader) {
it.header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})")
} else {
it
}
}
}
}
}

2
src/main/kotlin/mdnet/server/crypto.kt
View File

@ -20,7 +20,7 @@ package mdnet.server
import java.security.MessageDigest
fun md5Bytes(stringToHash: String) =
fun md5Bytes(stringToHash: String): ByteArray =
MessageDigest.getInstance("MD5")
.digest(stringToHash.toByteArray())

4
src/main/kotlin/mdnet/settings/ClientSettings.kt
View File

@ -47,7 +47,9 @@ data class ServerSettings(
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class DevSettings(
val devUrl: String? = null
val devUrl: String? = null,
val disableSniCheck: Boolean = false,
val sendServerHeader: Boolean = false,
)
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)

7
src/test/kotlin/mdnet/server/ImageServerTest.kt
View File

@ -39,6 +39,7 @@ import org.http4k.core.Response
import org.http4k.core.Status
import org.http4k.kotest.shouldHaveHeader
import org.http4k.kotest.shouldHaveStatus
import org.http4k.kotest.shouldNotHaveHeader
import org.http4k.routing.bind
import org.http4k.routing.routes
import org.ktorm.database.Database
@ -106,6 +107,12 @@ class ImageServerTest : FreeSpec() {
response.close()
}
}
"should not have Server header" {
val response = handler(Request(Method.GET, "/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
response.shouldNotHaveHeader("Server")
response.close()
}
}
"with real cache" - {