terribleplan/mangadex_at_home
1
0
Fork 0
forked from gitlab-com-mangadex-pub/mangadex_at_home
Code Issues Pull requests Releases Wiki Activity

merge

Browse source
This commit is contained in:
M 2020-06-12 12:58:10 -05:00
parent d7bf45af0b
commit 541890190a
5 changed files with 61 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/mdnet/base/Constants.java
View file

@ -3,7 +3,7 @@ package mdnet.base;
import java.time.Duration;
public class Constants {
public static final int CLIENT_BUILD = 3;
public static final int CLIENT_BUILD = 4;
public static final String CLIENT_VERSION = "1.0";
public static final Duration MAX_AGE_CACHE = Duration.ofDays(14);
}

2
src/main/java/mdnet/base/settings/ClientSettings.java
View file

@ -32,7 +32,7 @@ public final class ClientSettings {
}
public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond,
int clientPort, String clientSecret, int threads, WebSettings webSettings) {
int clientPort, String clientSecret, int threads, WebSettings webSettings) {
this.maxCacheSizeMib = maxCacheSizeMib;
this.maxBandwidthMibPerHour = maxBandwidthMibPerHour;
this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond;

8
src/main/java/mdnet/cache/DiskLruCache.java vendored
View file

@ -966,6 +966,10 @@ public final class DiskLruCache implements Closeable {
// Move files to new caching tree if exists
Path oldCache = Paths.get(directory + File.separator + key + "." + i);
Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i);
File newCacheDirectory = new File(directory + subKeyPath, key + "." + i + ".tmp");
newCacheDirectory.getParentFile().mkdirs();
if (Files.exists(oldCache)) {
try {
Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE);
@ -985,6 +989,10 @@ public final class DiskLruCache implements Closeable {
// Move files to new caching tree if exists
Path oldCache = Paths.get(directory + File.separator + key + "." + i + ".tmp");
Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i + ".tmp");
File newCacheDirectory = new File(directory + subKeyPath, key + "." + i + ".tmp");
newCacheDirectory.getParentFile().mkdirs();
if (Files.exists(oldCache)) {
try {
Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE);

98
src/main/kotlin/mdnet/base/Application.kt
View file

@ -19,10 +19,8 @@ import org.http4k.filter.CachingFilters
import org.http4k.filter.MaxAgeTtl
import org.http4k.filter.ServerFilters
import org.http4k.lens.Path
import org.http4k.routing.ResourceLoader
import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.routing.singlePageApp
import org.http4k.server.Http4kServer
import org.http4k.server.asServer
import org.slf4j.LoggerFactory
@ -52,16 +50,16 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
}
val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom()
.setDefaultRequestConfig(RequestConfig.custom()
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.setConnectionRequestTimeout(3000)
.setDefaultRequestConfig(RequestConfig.custom()
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.setConnectionRequestTimeout(3000)
.build())
.setMaxConnTotal(THREADS_TO_ALLOCATE)
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
// Have it at the maximum open sockets a user can have in most modern OSes. No reason to limit this, just limit it at the Netty side.
.build())
.setMaxConnTotal(THREADS_TO_ALLOCATE)
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
// Have it at the maximum open sockets a user can have in most modern OSes. No reason to limit this, just limit it at the Netty side.
.build())
val app = { dataSaver: Boolean ->
{ request: Request ->
@ -89,28 +87,28 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
// Netty doesn't do Content-Length or Content-Type, so we have the pleasure of doing that ourselves
fun respondWithImage(input: InputStream, length: String?, type: String, lastModified: String?): Response =
Response(Status.OK)
.header("Content-Type", type)
.header("X-Content-Type-Options", "nosniff")
.header(
"Cache-Control",
listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
)
.header("Timing-Allow-Origin", "https://mangadex.org")
.let {
if (length != null) {
it.body(input, length.toLong()).header("Content-Length", length)
} else {
it.body(input).header("Transfer-Encoding", "chunked")
}
}
.let {
if (lastModified != null) {
it.header("Last-Modified", lastModified)
} else {
it
}
}
Response(Status.OK)
.header("Content-Type", type)
.header("X-Content-Type-Options", "nosniff")
.header(
"Cache-Control",
listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
)
.header("Timing-Allow-Origin", "https://mangadex.org")
.let {
if (length != null) {
it.body(input, length.toLong()).header("Content-Length", length)
} else {
it.body(input).header("Transfer-Encoding", "chunked")
}
}
.let {
if (lastModified != null) {
it.header("Last-Modified", lastModified)
} else {
it
}
}
val snapshot = cache.get(cacheId)
if (snapshot != null) {
@ -126,15 +124,15 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
snapshot.close()
Response(Status.NOT_MODIFIED)
.header("Last-Modified", lastModified)
.header("Last-Modified", lastModified)
} else {
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri hit cache")
}
respondWithImage(
CipherInputStream(BufferedInputStream(snapshot.getInputStream(0)), getRc4(rc4Bytes)),
snapshot.getLength(0).toString(), snapshot.getString(1), snapshot.getString(2)
CipherInputStream(BufferedInputStream(snapshot.getInputStream(0)), getRc4(rc4Bytes)),
snapshot.getLength(0).toString(), snapshot.getString(1), snapshot.getString(2)
)
}
} else {
@ -171,8 +169,8 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
editor.setString(2, lastModified)
val tee = CachingInputStream(
mdResponse.body.stream,
executor, CipherOutputStream(BufferedOutputStream(editor.newOutputStream(0)), getRc4(rc4Bytes))
mdResponse.body.stream,
executor, CipherOutputStream(BufferedOutputStream(editor.newOutputStream(0)), getRc4(rc4Bytes))
) {
// Note: if neither of the options get called/are in the log
// check that tee gets closed and for exceptions in this lambda
@ -207,19 +205,17 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
CachingFilters
return catchAllHideDetails()
.then(ServerFilters.CatchLensFailure)
.then(addCommonHeaders())
.then(
routes(
"/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true),
"/{token}/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true),
singlePageApp(ResourceLoader.Classpath("/webui"))
.then(ServerFilters.CatchLensFailure)
.then(addCommonHeaders())
.then(
routes(
"/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true),
"/{token}/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true)
)
)
)
.asServer(Netty(serverSettings.tls, clientSettings, statistics))
.asServer(Netty(serverSettings.tls, clientSettings, statistics))
}
private fun getRc4(key: ByteArray): Cipher {
@ -235,7 +231,7 @@ private fun addCommonHeaders(): Filter {
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
}
}
}

8
src/main/kotlin/mdnet/base/Netty.kt
View file

@ -46,7 +46,7 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
private lateinit var address: InetSocketAddress
private val burstLimiter = object : GlobalTrafficShapingHandler(
workerGroup, 1024 * clientSettings.maxBurstRateKibPerSecond, 0, 50) {
workerGroup, 1024 * clientSettings.maxBurstRateKibPerSecond, 0, 50) {
override fun doAccounting(counter: TrafficCounter) {
stats.get().bytesSent.getAndAdd(counter.cumulativeWrittenBytes())
counter.resetCumulativeTime()
@ -60,9 +60,9 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
val (mainCert, chainCert) = getX509Certs(tls.certificate)
val sslContext = SslContextBuilder
.forServer(getPrivateKey(tls.privateKey), mainCert, chainCert)
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
.build()
.forServer(getPrivateKey(tls.privateKey), mainCert, chainCert)
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
.build()
val bootstrap = ServerBootstrap()
bootstrap.group(masterGroup, workerGroup)