400 lines
16 KiB
Kotlin
400 lines
16 KiB
Kotlin
/*
|
|
Mangadex@Home
|
|
Copyright (c) 2020, MangaDex Network
|
|
This file is part of MangaDex@Home.
|
|
|
|
MangaDex@Home is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
MangaDex@Home is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
package mdnet.server
|
|
|
|
import com.fasterxml.jackson.core.JsonProcessingException
|
|
import com.fasterxml.jackson.databind.DeserializationFeature
|
|
import com.fasterxml.jackson.databind.ObjectMapper
|
|
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
|
|
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
|
import com.fasterxml.jackson.module.kotlin.readValue
|
|
import io.micrometer.core.instrument.FunctionCounter
|
|
import io.micrometer.core.instrument.Timer
|
|
import io.micrometer.prometheus.PrometheusMeterRegistry
|
|
import mdnet.Constants
|
|
import mdnet.cache.CachingInputStream
|
|
import mdnet.cache.Image
|
|
import mdnet.cache.ImageMetadata
|
|
import mdnet.cache.ImageStorage
|
|
import mdnet.data.Statistics
|
|
import mdnet.data.Token
|
|
import mdnet.logging.info
|
|
import mdnet.logging.trace
|
|
import mdnet.logging.warn
|
|
import mdnet.metrics.GeoIpMetricsFilterBuilder
|
|
import mdnet.metrics.PostTransactionLabeler
|
|
import mdnet.netty.Netty
|
|
import mdnet.security.TweetNaclFast
|
|
import mdnet.settings.MetricsSettings
|
|
import mdnet.settings.RemoteSettings
|
|
import mdnet.settings.ServerSettings
|
|
import org.apache.hc.client5.http.config.RequestConfig
|
|
import org.apache.hc.client5.http.cookie.StandardCookieSpec
|
|
import org.apache.hc.client5.http.impl.classic.HttpClients
|
|
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
|
|
import org.apache.hc.core5.util.Timeout
|
|
import org.http4k.client.ApacheClient
|
|
import org.http4k.core.*
|
|
import org.http4k.filter.CachingFilters
|
|
import org.http4k.filter.ClientFilters
|
|
import org.http4k.filter.MicrometerMetrics
|
|
import org.http4k.filter.ServerFilters
|
|
import org.http4k.lens.LensFailure
|
|
import org.http4k.lens.Path
|
|
import org.http4k.routing.bind
|
|
import org.http4k.routing.routes
|
|
import org.http4k.server.Http4kServer
|
|
import org.http4k.server.asServer
|
|
import org.slf4j.LoggerFactory
|
|
import java.io.BufferedInputStream
|
|
import java.io.BufferedOutputStream
|
|
import java.io.InputStream
|
|
import java.time.Clock
|
|
import java.time.OffsetDateTime
|
|
import java.util.*
|
|
import java.util.concurrent.Executors
|
|
import java.util.concurrent.atomic.AtomicReference
|
|
|
|
private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
|
|
private val JACKSON: ObjectMapper = jacksonObjectMapper()
|
|
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
|
.registerModule(JavaTimeModule())
|
|
|
|
class ImageServer(
|
|
private val storage: ImageStorage,
|
|
private val statistics: AtomicReference<Statistics>,
|
|
private val client: HttpHandler,
|
|
registry: PrometheusMeterRegistry
|
|
) {
|
|
private val executor = Executors.newCachedThreadPool()
|
|
private val cacheLookupTimer = Timer
|
|
.builder("cache_lookup")
|
|
.publishPercentiles(0.5, 0.75, 0.9, 0.99)
|
|
.register(registry)
|
|
|
|
// This is part of the ImageServer, and it expects `chapterHash` and `fileName` path segments.
|
|
fun handler(dataSaver: Boolean): HttpHandler = baseHandler().then { request ->
|
|
val chapterHash = Path.of("chapterHash")(request)
|
|
val fileName = Path.of("fileName")(request)
|
|
|
|
val sanitizedUri = if (dataSaver) {
|
|
"/data-saver"
|
|
} else {
|
|
"/data"
|
|
} + "/$chapterHash/$fileName"
|
|
|
|
val imageId = if (dataSaver) {
|
|
md5Bytes("saver$chapterHash.$fileName")
|
|
} else {
|
|
md5Bytes("$chapterHash.$fileName")
|
|
}.let {
|
|
printHexString(it)
|
|
}
|
|
|
|
val image: Image? = cacheLookupTimer.recordCallable { storage.loadImage(imageId) }
|
|
|
|
if (image != null) {
|
|
request.handleCacheHit(sanitizedUri, image)
|
|
} else {
|
|
request.handleCacheMiss(sanitizedUri, imageId)
|
|
}
|
|
}
|
|
|
|
private fun Request.handleCacheHit(sanitizedUri: String, image: Image): Response {
|
|
// our files never change, so it's safe to use the browser cache
|
|
return if (this.header("If-Modified-Since") != null) {
|
|
LOGGER.info { "Request for $sanitizedUri cached by browser" }
|
|
|
|
val lastModified = image.data.lastModified
|
|
|
|
Response(Status.NOT_MODIFIED)
|
|
.header("Last-Modified", lastModified)
|
|
} else {
|
|
LOGGER.info { "Request for $sanitizedUri hit cache" }
|
|
|
|
respondWithImage(
|
|
BufferedInputStream(image.stream),
|
|
image.data.size, image.data.contentType, image.data.lastModified,
|
|
true
|
|
)
|
|
}
|
|
}
|
|
|
|
private fun Request.handleCacheMiss(sanitizedUri: String, imageId: String): Response {
|
|
LOGGER.info { "Request for $sanitizedUri missed cache" }
|
|
|
|
val mdResponse = client(Request(Method.GET, sanitizedUri))
|
|
|
|
if (mdResponse.status != Status.OK) {
|
|
LOGGER.trace { "Upstream query for $sanitizedUri errored with status ${mdResponse.status}" }
|
|
|
|
mdResponse.close()
|
|
return Response(mdResponse.status)
|
|
}
|
|
|
|
val contentType = mdResponse.header("Content-Type")!!
|
|
val contentLength = mdResponse.header("Content-Length")?.toInt()
|
|
val lastModified = mdResponse.header("Last-Modified")
|
|
|
|
if (!contentType.isImageMimetype()) {
|
|
LOGGER.warn { "Upstream query for $sanitizedUri returned bad mimetype $contentType" }
|
|
mdResponse.close()
|
|
return Response(Status.INTERNAL_SERVER_ERROR)
|
|
}
|
|
|
|
// bad upstream responses mean we can't cache, so bail
|
|
if (contentLength == null || lastModified == null) {
|
|
LOGGER.trace { "Request for $sanitizedUri is being served due to upstream issues" }
|
|
return respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
|
|
}
|
|
|
|
LOGGER.trace { "Upstream query for $sanitizedUri succeeded" }
|
|
|
|
val writer = storage.storeImage(imageId, ImageMetadata(contentType, lastModified, contentLength))
|
|
|
|
// A null writer means that this file is being written to
|
|
// concurrently so we skip the cache process
|
|
return if (writer != null) {
|
|
LOGGER.trace { "Request for $sanitizedUri is being cached and served" }
|
|
|
|
val tee = CachingInputStream(
|
|
mdResponse.body.stream,
|
|
executor, BufferedOutputStream(writer.stream),
|
|
) {
|
|
try {
|
|
if (writer.commit(contentLength)) {
|
|
LOGGER.info { "Cache download for $sanitizedUri committed" }
|
|
} else {
|
|
LOGGER.warn { "Cache download for $sanitizedUri aborted" }
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Cache go/no go for $sanitizedUri failed" }
|
|
}
|
|
}
|
|
respondWithImage(tee, contentLength, contentType, lastModified, false)
|
|
} else {
|
|
LOGGER.trace { "Request for $sanitizedUri is being served" }
|
|
respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
|
|
}
|
|
}
|
|
|
|
private fun respondWithImage(input: InputStream, length: Int?, type: String, lastModified: String?, cached: Boolean): Response =
|
|
Response(Status.OK)
|
|
.header("Content-Type", type)
|
|
.header("X-Content-Type-Options", "nosniff")
|
|
.let {
|
|
if (length != null) {
|
|
it.body(input, length.toLong()).header("Content-Length", length.toString())
|
|
} else {
|
|
it.body(input).header("Transfer-Encoding", "chunked")
|
|
}
|
|
}
|
|
.let {
|
|
if (lastModified != null) {
|
|
it.header("Last-Modified", lastModified)
|
|
} else {
|
|
it
|
|
}
|
|
}
|
|
.header("X-Cache", if (cached) "HIT" else "MISS")
|
|
|
|
companion object {
|
|
private fun baseHandler(): Filter =
|
|
CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE)
|
|
.then { next: HttpHandler ->
|
|
{ request: Request ->
|
|
val response = next(request)
|
|
response.header("access-control-allow-origin", "https://mangadex.org")
|
|
.header("access-control-allow-headers", "*")
|
|
.header("access-control-allow-methods", "GET")
|
|
.header("timing-allow-origin", "https://mangadex.org")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/")
|
|
|
|
fun getServer(
|
|
storage: ImageStorage,
|
|
remoteSettings: RemoteSettings,
|
|
serverSettings: ServerSettings,
|
|
statistics: AtomicReference<Statistics>,
|
|
metricsSettings: MetricsSettings,
|
|
registry: PrometheusMeterRegistry,
|
|
): Http4kServer {
|
|
val apache = ApacheClient(
|
|
responseBodyMode = BodyMode.Stream,
|
|
client = HttpClients.custom()
|
|
.disableConnectionState()
|
|
.setDefaultRequestConfig(
|
|
RequestConfig.custom()
|
|
.setCookieSpec(StandardCookieSpec.IGNORE)
|
|
.setConnectTimeout(Timeout.ofSeconds(2))
|
|
.setResponseTimeout(Timeout.ofSeconds(2))
|
|
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
|
|
.build()
|
|
)
|
|
.setConnectionManager(
|
|
PoolingHttpClientConnectionManagerBuilder.create()
|
|
.setMaxConnTotal(3000)
|
|
.setMaxConnPerRoute(100)
|
|
.build()
|
|
)
|
|
.build()
|
|
)
|
|
|
|
val client =
|
|
ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
|
|
.then(ClientFilters.MicrometerMetrics.RequestCounter(registry))
|
|
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
|
|
.then(apache)
|
|
|
|
val imageServer = ImageServer(
|
|
storage = storage,
|
|
statistics = statistics,
|
|
client = client,
|
|
registry = registry
|
|
)
|
|
|
|
FunctionCounter.builder(
|
|
"client_sent_bytes",
|
|
statistics,
|
|
{ it.get().bytesSent.toDouble() }
|
|
).register(registry)
|
|
|
|
val verifier = tokenVerifier(
|
|
tokenKey = remoteSettings.tokenKey,
|
|
shouldVerify = { chapter, _ ->
|
|
!remoteSettings.forceDisableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e")
|
|
}
|
|
)
|
|
|
|
return addCommonHeaders()
|
|
.then(timeRequest())
|
|
.then(catchAllHideDetails())
|
|
.then(
|
|
routes(
|
|
"/{token}/data/{chapterHash}/{fileName}" bind Method.GET to verifier.then(
|
|
imageServer.handler(
|
|
dataSaver = false,
|
|
)
|
|
),
|
|
"/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to verifier.then(
|
|
imageServer.handler(
|
|
dataSaver = true,
|
|
)
|
|
),
|
|
"/data/{chapterHash}/{fileName}" bind Method.GET to verifier.then(
|
|
imageServer.handler(
|
|
dataSaver = false,
|
|
)
|
|
),
|
|
"/data-saver/{chapterHash}/{fileName}" bind Method.GET to verifier.then(
|
|
imageServer.handler(
|
|
dataSaver = true,
|
|
)
|
|
),
|
|
"/prometheus" bind Method.GET to {
|
|
Response(Status.OK).body(registry.scrape())
|
|
}
|
|
).withFilter(
|
|
ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
|
|
).withFilter(
|
|
GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build()
|
|
)
|
|
)
|
|
.asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))
|
|
}
|
|
|
|
fun timeRequest(): Filter {
|
|
return Filter { next: HttpHandler ->
|
|
{ request: Request ->
|
|
val cleanedUri = request.uri.path.replaceBefore("/data", "/{token}")
|
|
|
|
LOGGER.info { "Request for $cleanedUri received" }
|
|
|
|
val start = System.currentTimeMillis()
|
|
val response = next(request)
|
|
val latency = System.currentTimeMillis() - start
|
|
|
|
LOGGER.info { "Request for $cleanedUri completed (TTFB) in ${latency}ms" }
|
|
|
|
response
|
|
}
|
|
}
|
|
}
|
|
|
|
fun tokenVerifier(tokenKey: ByteArray, shouldVerify: (String, String) -> Boolean): Filter {
|
|
val box = TweetNaclFast.SecretBox(tokenKey)
|
|
|
|
return Filter { next ->
|
|
then@{
|
|
val chapterHash = Path.of("chapterHash")(it)
|
|
val fileName = Path.of("fileName")(it)
|
|
|
|
if (shouldVerify(chapterHash, fileName)) {
|
|
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
|
|
|
|
val tokenArr = try {
|
|
val toDecode = try {
|
|
Path.of("token")(it)
|
|
} catch (e: LensFailure) {
|
|
LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" }
|
|
return@then Response(Status.FORBIDDEN).body("Token is missing")
|
|
}
|
|
Base64.getUrlDecoder().decode(toDecode)
|
|
} catch (e: IllegalArgumentException) {
|
|
LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" }
|
|
return@then Response(Status.FORBIDDEN).body("Token is invalid base64")
|
|
}
|
|
if (tokenArr.size < 24) {
|
|
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
|
return@then Response(Status.FORBIDDEN)
|
|
}
|
|
val token = try {
|
|
JACKSON.readValue<Token>(
|
|
box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply {
|
|
if (this == null) {
|
|
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
|
return@then Response(Status.FORBIDDEN)
|
|
}
|
|
}
|
|
)
|
|
} catch (e: JsonProcessingException) {
|
|
LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" }
|
|
return@then Response(Status.FORBIDDEN).body("Token is invalid")
|
|
}
|
|
|
|
if (OffsetDateTime.now().isAfter(token.expires)) {
|
|
LOGGER.info { "Request for $cleanedUri rejected for expired token" }
|
|
return@then Response(Status.GONE).body("Token has expired")
|
|
}
|
|
|
|
if (token.hash != chapterHash) {
|
|
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
|
|
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
|
|
}
|
|
}
|
|
|
|
return@then next(it)
|
|
}
|
|
}
|
|
}
|