terribleplan
/
mangadex_at_home
forked from gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reorganize some code - add read and write timeouts

This commit is contained in:
carbotaniuman 2020-06-30 14:06:12 -05:00
parent 63c810815f
commit 3445f5d569
12 changed files with 75 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- [2020-06-23] Added Gitlab CI integration by [@lflare].
- [2020-06-28] Added `client_external_port setting` [@wedge1001].
- [2020-06-29] Added rudimentary support of Referer checking to mitigate hotlinking by [@lflare].
- [2020-06-30] Added read and write timeouts to protect against some attacks [@carbotaniuman].
### Changed

5
build.gradle
View File

@ -30,12 +30,15 @@ dependencies {
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
runtimeOnly group:"io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final"
implementation group:"ch.qos.logback", name: "logback-classic", version: "1.2.1"
implementation group:"ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha5"
implementation group: "org.jetbrains.exposed", name: "exposed-core", version: "$exposed_version"
implementation group: "org.jetbrains.exposed", name: "exposed-dao", version: "$exposed_version"
implementation group: "org.jetbrains.exposed", name: "exposed-jdbc", version: "$exposed_version"
implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1"
implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
implementation "net.java.dev.jna:jna:5.5.0"
}
java {

3
src/main/kotlin/mdnet/base/Constants.kt
View File

@ -29,4 +29,7 @@ object Constants {
const val WEBUI_VERSION = "0.1.1"
val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true)
const val MAX_READ_TIME_SECONDS = 300
const val MAX_WRITE_TIME_SECONDS = 60
}

5
src/main/kotlin/mdnet/base/MangaDexClient.kt
View File

@ -23,6 +23,7 @@ import ch.qos.logback.classic.LoggerContext
import com.fasterxml.jackson.module.kotlin.readValue
import mdnet.base.Constants.JACKSON
import mdnet.base.Main.dieWithError
import mdnet.base.data.Statistics
import mdnet.base.server.getServer
import mdnet.base.server.getUiServer
import mdnet.base.settings.ClientSettings
@ -64,7 +65,9 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
return this.size > 240
}
})
private val statistics: AtomicReference<Statistics> = AtomicReference(Statistics())
private val statistics: AtomicReference<Statistics> = AtomicReference(
Statistics()
)
private val isHandled: AtomicBoolean = AtomicBoolean(false)
private var webUi: Http4kServer? = null
private val cache: DiskLruCache

2
src/main/kotlin/mdnet/base/Statistics.kt → src/main/kotlin/mdnet/base/data/Statistics.kt
View File

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base
package mdnet.base.data
import com.fasterxml.jackson.databind.PropertyNamingStrategy
import com.fasterxml.jackson.databind.annotation.JsonNaming

2
src/main/kotlin/mdnet/base/dao/metadata.kt → src/main/kotlin/mdnet/base/data/metadata.kt
View File

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.dao
package mdnet.base.data
import org.jetbrains.exposed.dao.Entity
import org.jetbrains.exposed.dao.EntityClass

22
src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt
View File

@ -16,28 +16,24 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.netty
import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.ChannelFactory
import io.netty.channel.ChannelFuture
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelInboundHandlerAdapter
import io.netty.channel.ChannelInitializer
import io.netty.channel.ChannelOption
import io.netty.channel.ServerChannel
import io.netty.channel.*
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.SocketChannel
import io.netty.channel.socket.nio.NioServerSocketChannel
import io.netty.handler.codec.DecoderException
import io.netty.handler.codec.http.HttpObjectAggregator
import io.netty.handler.codec.http.HttpServerCodec
import io.netty.handler.codec.http.HttpServerKeepAliveHandler
import io.netty.handler.codec.http.*
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.stream.ChunkedWriteHandler
import io.netty.handler.timeout.ReadTimeoutHandler
import io.netty.handler.timeout.WriteTimeoutHandler
import io.netty.handler.traffic.GlobalTrafficShapingHandler
import io.netty.handler.traffic.TrafficCounter
import mdnet.base.Statistics
import mdnet.base.Constants
import mdnet.base.data.Statistics
import mdnet.base.settings.ClientSettings
import mdnet.base.settings.TlsCert
import org.http4k.core.HttpHandler
@ -99,6 +95,10 @@ class Netty(private val tls: TlsCert, private val clientSettings: ClientSettings
ch.pipeline().addLast("aggregator", HttpObjectAggregator(65536))
ch.pipeline().addLast("burstLimiter", burstLimiter)
ch.pipeline().addLast("readTimeoutHandler", ReadTimeoutHandler(Constants.MAX_READ_TIME_SECONDS))
ch.pipeline().addLast("writeTimeoutHandler", WriteTimeoutHandler(Constants.MAX_WRITE_TIME_SECONDS))
ch.pipeline().addLast("streamer", ChunkedWriteHandler())
ch.pipeline().addLast("handler", Http4kChannelHandler(httpHandler))

3
src/main/kotlin/mdnet/base/netty/WebUiNetty.kt
View File

@ -37,9 +37,6 @@ import org.http4k.core.HttpHandler
import org.http4k.server.Http4kChannelHandler
import org.http4k.server.Http4kServer
import org.http4k.server.ServerConfig
import org.slf4j.LoggerFactory
private val LOGGER = LoggerFactory.getLogger(WebUiNetty::class.java)
class WebUiNetty(private val hostname: String, private val port: Int) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {

26
src/main/kotlin/mdnet/base/server/Application.kt
View File

@ -21,20 +21,22 @@ package mdnet.base.server
import mdnet.base.netty.Netty
import mdnet.base.settings.ServerSettings
import mdnet.base.Statistics
import mdnet.base.data.Statistics
import mdnet.base.settings.ClientSettings
import mdnet.cache.DiskLruCache
import org.http4k.core.Method
import org.http4k.core.then
import org.http4k.core.*
import org.http4k.filter.ServerFilters
import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.server.Http4kServer
import org.http4k.server.asServer
import org.jetbrains.exposed.sql.Database
import org.slf4j.LoggerFactory
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
private val LOGGER = LoggerFactory.getLogger("Application")
fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled)
@ -59,3 +61,21 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
)
.asServer(Netty(serverSettings.tls!!, clientSettings, statistics))
}
fun timeRequest(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val start = System.currentTimeMillis()
val response = next(request)
val latency = System.currentTimeMillis() - start
if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
val sanitizedUri = response.header("X-Uri")
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
}
}
response.header("X-Time-Taken", latency.toString())
}
}
}

42
src/main/kotlin/mdnet/base/server/ImageServer.kt
View File

@ -20,9 +20,9 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base.server
import mdnet.base.Constants
import mdnet.base.Statistics
import mdnet.base.dao.ImageData
import mdnet.base.dao.ImageDatum
import mdnet.base.data.Statistics
import mdnet.base.data.ImageData
import mdnet.base.data.ImageDatum
import mdnet.cache.CachingInputStream
import mdnet.cache.DiskLruCache
import org.apache.http.client.config.CookieSpecs
@ -30,7 +30,9 @@ import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.HttpClients
import org.http4k.client.ApacheClient
import org.http4k.core.*
import org.http4k.filter.MaxAgeTtl
import org.http4k.filter.CachingFilters
import org.http4k.filter.CorsPolicy
import org.http4k.filter.ServerFilters
import org.http4k.lens.Path
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.SchemaUtils
@ -41,6 +43,7 @@ import java.io.BufferedOutputStream
import java.io.File
import java.io.InputStream
import java.security.MessageDigest
import java.time.Clock
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
@ -70,7 +73,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
.build())
fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = { request ->
fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request ->
val chapterHash = Path.of("chapterHash")(request)
val fileName = Path.of("fileName")(request)
@ -105,10 +108,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
handled.set(true)
if (referer != null && !referer.startsWith("https://mangadex.org")) {
if (snapshot != null) {
snapshot.close()
}
snapshot?.close()
Response(Status.FORBIDDEN)
} else if (snapshot != null && imageDatum != null) {
request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
@ -255,13 +255,6 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
Response(Status.OK)
.header("Content-Type", type)
.header("X-Content-Type-Options", "nosniff")
.header(
"Cache-Control",
listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
)
.header("Access-Control-Allow-Origin", "https://mangadex.org")
.header("Access-Control-Expose-Headers", "*")
.header("Timing-Allow-Origin", "https://mangadex.org")
.let {
if (length != null) {
it.body(input, length.toLong()).header("Content-Length", length)
@ -280,6 +273,23 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
companion object {
private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
private fun baseHandler(): Filter =
CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE)
.then(ServerFilters.Cors(
CorsPolicy(
origins = listOf("https://mangadex.org"),
headers = listOf("*"),
methods = Method.values().toList()
)
)
)
.then(Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("timing-allow-origin", "https://mangadex.org")
}
})
}
}

2
src/main/kotlin/mdnet/base/server/WebUi.kt
View File

@ -21,7 +21,7 @@ package mdnet.base.server
import java.time.Instant
import java.util.concurrent.atomic.AtomicReference
import mdnet.base.Statistics
import mdnet.base.data.Statistics
import mdnet.base.netty.WebUiNetty
import mdnet.base.settings.WebSettings
import org.http4k.core.Body

22
src/main/kotlin/mdnet/base/server/common.kt
View File

@ -58,25 +58,3 @@ fun catchAllHideDetails(): Filter {
}
}
}
fun timeRequest(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val start = System.currentTimeMillis()
val response = next(request)
val latency = System.currentTimeMillis() - start
if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
// Dirty hack to get sanitizedUri from ImageServer
val sanitizedUri = response.header("X-Uri")
// Log in TRACE
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
}
// Delete response header entirely
response.header("X-Uri", null)
}
// Set response header with processing times
response.header("X-Time-Taken", latency.toString())
}
}
}