Merge branch 'master' into expermential_buffer_size_knob

This commit is contained in:
AviKav 2020-08-04 22:09:18 -04:00
commit 7852030f8f
No known key found for this signature in database
GPG key ID: 7BC4B96BC8A7167D
7 changed files with 333 additions and 96 deletions

View file

@ -1,6 +1,7 @@
stages:
- build
- publish
- publish_latest
- publish_docker
cache:
@ -26,8 +27,22 @@ publish:
name: "mangadex_at_home"
paths:
- "*.jar"
- "mangadex_at_home-*.zip"
- settings.sample.json
publish_latest:
image: alpine
stage: publish
before_script:
- apk update && apk add git
- export VERSION=`git describe --tags --dirty`
script:
- cp build/libs/mangadex_at_home-${VERSION}-all.jar build/libs/mangadex_at_home-latest-all.jar
artifacts:
name: "mangadex_at_home-latest"
paths:
- "build/libs/mangadex_at_home-latest-all.jar"
publish_docker:
image: docker:git
stage: publish

View file

@ -6,14 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- [2020-07-13] Added reloading client setting without stopping client by [@radonbark].
### Changed
- [2020-07-29] Disallow unsafe ports [@m3ch_mania].
### Deprecated
### Removed
### Fixed
- [2020-07-29] Fixed stupid libsodium bugs [@carbotaniuman].
- [2020-07-29] Fixed issues from the Great Cache Propagation [@carbotaniuman].
### Security

View file

@ -22,8 +22,83 @@ import java.time.Duration
object Constants {
const val CLIENT_BUILD = 16
val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
const val MAX_READ_TIME_SECONDS = 300
const val MAX_WRITE_TIME_SECONDS = 60
// General list of ports to which Firefox and Chromium will not send HTTP requests for security reasons
// See:
// * https://chromium.googlesource.com/chromium/src.git/+/refs/heads/master/net/base/port_util.cc
// * https://developer.mozilla.org/en-US/docs/Mozilla/Mozilla_Port_Blocking#Blocked_Ports
@JvmField val RESTRICTED_PORTS = intArrayOf(
1, // tcpmux
7, // echo
9, // discard
11, // systat
13, // daytime
15, // netstat
17, // qotd
19, // chargen
20, // ftp data
21, // ftp access
22, // ssh
23, // telnet
25, // smtp
37, // time
42, // name
43, // nicname
53, // domain
77, // priv-rjs
79, // finger
87, // ttylink
95, // supdup
101, // hostriame
102, // iso-tsap
103, // gppitnp
104, // acr-nema
109, // pop2
110, // pop3
111, // sunrpc
113, // auth
115, // sftp
117, // uucp-path
119, // nntp
123, // NTP
135, // loc-srv /epmap
139, // netbios
143, // imap2
179, // BGP
389, // ldap
427, // SLP (Also used by Apple Filing Protocol)
465, // smtp+ssl
512, // print / exec
513, // login
514, // shell
515, // printer
526, // tempo
530, // courier
531, // chat
532, // netnews
540, // uucp
548, // AFP (Apple Filing Protocol)
556, // remotefs
563, // nntp+ssl
587, // smtp (rfc6409)
601, // syslog-conn (rfc3195)
636, // ldap+ssl
993, // ldap+ssl
995, // pop3+ssl
2049, // nfs
3659, // apple-sasl / PasswordServer
4045, // lockd
6000, // X11
6665, // Alternate IRC [Apple addition]
6666, // Alternate IRC [Apple addition]
6667, // Standard IRC [Apple addition]
6668, // Alternate IRC [Apple addition]
6669, // Alternate IRC [Apple addition]
6697 // IRC + TLS
)
}

View file

@ -19,25 +19,12 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base
import ch.qos.logback.classic.LoggerContext
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.FileReader
import java.io.FileWriter
import java.io.IOException
import java.util.regex.Pattern
import kotlin.system.exitProcess
import mdnet.BuildInfo
import mdnet.base.settings.ClientSettings
import org.slf4j.LoggerFactory
object Main {
private val LOGGER = LoggerFactory.getLogger(Main::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true)
@JvmStatic
fun main(args: Array<String>) {
@ -68,25 +55,7 @@ object Main {
dieWithError("Expected one argument: path to config file, or nothing")
}
val settings = try {
JACKSON.readValue<ClientSettings>(FileReader(file))
} catch (e: UnrecognizedPropertyException) {
dieWithError("'${e.propertyName}' is not a valid setting")
} catch (e: JsonProcessingException) {
dieWithError(e)
} catch (ignored: IOException) {
ClientSettings().also {
LOGGER.warn("Settings file {} not found, generating file", file)
try {
FileWriter(file).use { writer -> JACKSON.writeValue(writer, it) }
} catch (e: IOException) {
dieWithError(e)
}
}
}.apply(::validateSettings)
LOGGER.info { "Client settings loaded: $settings" }
val client = MangaDexClient(settings)
val client = MangaDexClient(file)
Runtime.getRuntime().addShutdownHook(Thread { client.shutdown() })
client.runLoop()
}
@ -103,40 +72,4 @@ object Main {
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
private fun validateSettings(settings: ClientSettings) {
if (!isSecretValid(settings.clientSecret)) dieWithError("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
if (settings.clientPort == 0) {
dieWithError("Config Error: Invalid port number")
}
if (settings.maxCacheSizeInMebibytes < 1024) {
dieWithError("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
}
if (settings.threads < 4) {
dieWithError("Config Error: Invalid number of threads, must be >= 4")
}
if (settings.maxMebibytesPerHour < 0) {
dieWithError("Config Error: Max bandwidth must be >= 0")
}
if (settings.maxKilobitsPerSecond < 0) {
dieWithError("Config Error: Max burst rate must be >= 0")
}
if (settings.gracefulShutdownWaitSeconds < 15) {
dieWithError("Config Error: Graceful shutdown wait be >= 15")
}
if (settings.webSettings != null) {
if (settings.webSettings.uiPort == 0) {
dieWithError("Config Error: Invalid UI port number")
}
}
if (settings.experimental != null) {
if (settings.experimental.maxBufferSizeForCacheHit < 0)
dieWithError("Config Error: Max cache buffer multiple must be >= 0")
}
}
private const val CLIENT_KEY_LENGTH = 52
private fun isSecretValid(clientSecret: String): Boolean {
return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)
}
}

View file

@ -20,10 +20,14 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base
import ch.qos.logback.classic.LoggerContext
import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.IOException
import java.time.Instant
import java.util.*
@ -32,6 +36,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
import java.util.regex.Pattern
import mdnet.base.Main.dieWithError
import mdnet.base.data.Statistics
import mdnet.base.server.getServer
@ -43,23 +48,28 @@ import mdnet.cache.HeaderMismatchException
import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory
private const val CLIENT_KEY_LENGTH = 52
// Exception class to handle when Client Settings have invalid values
class ClientSettingsException(message: String) : Exception(message)
sealed class State
// server is not running
object Uninitialized : State()
data class Uninitialized(val clientSettings: ClientSettings) : State()
// server has shut down
object Shutdown : State()
// server is in the process of shutting down
data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized, val action: () -> Unit = {}) : State()
data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized(lastRunning.clientSettings), val action: () -> Unit = {}) : State()
// server is currently running
data class Running(val server: Http4kServer, val settings: ServerSettings) : State()
class MangaDexClient(private val clientSettings: ClientSettings) {
data class Running(val server: Http4kServer, val settings: ServerSettings, val clientSettings: ClientSettings) : State()
// clientSettings must only be accessed from the thread on the executorService
class MangaDexClient(private val clientSettingsFile: String) {
// this must remain singlethreaded because of how the state mechanism works
private val executorService = Executors.newSingleThreadScheduledExecutor()
// state must only be accessed from the thread on the executorService
private var state: State = Uninitialized
private var state: State
private val serverHandler: ServerHandler = ServerHandler(clientSettings)
private var serverHandler: ServerHandler
private val statsMap: MutableMap<Instant, Statistics> = Collections
.synchronizedMap(object : LinkedHashMap<Instant, Statistics>(240) {
override fun removeEldestEntry(eldest: Map.Entry<Instant, Statistics>): Boolean {
@ -74,6 +84,32 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
private val cache: DiskLruCache
init {
// Read ClientSettings
val clientSettings = try {
readClientSettings()
} catch (e: UnrecognizedPropertyException) {
dieWithError("'${e.propertyName}' is not a valid setting")
} catch (e: JsonProcessingException) {
dieWithError(e)
} catch (ignored: IOException) {
ClientSettings().also {
LOGGER.warn { "Settings file $clientSettingsFile not found, generating file" }
try {
FileWriter(clientSettingsFile).use { writer -> JACKSON.writeValue(writer, it) }
} catch (e: IOException) {
dieWithError(e)
}
}
} catch (e: ClientSettingsException) {
dieWithError(e)
}
// Initialize things that depend on Client Settings
LOGGER.info { "Client settings loaded: $clientSettings" }
state = Uninitialized(clientSettings)
serverHandler = ServerHandler(clientSettings)
// Initialize everything else
try {
cache = DiskLruCache.open(
File("cache"), 1, 1,
@ -94,11 +130,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
fun runLoop() {
loginAndStartServer()
statsMap[Instant.now()] = statistics.get()
if (clientSettings.webSettings != null) {
webUi = getUiServer(clientSettings.webSettings, statistics, statsMap)
webUi!!.start()
}
startWebUi()
LOGGER.info { "Mangadex@Home Client initialized. Starting normal operation." }
executorService.scheduleAtFixedRate({
@ -140,11 +172,11 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}, 1, 1, TimeUnit.HOURS)
val timesToWait = clientSettings.gracefulShutdownWaitSeconds / 15
executorService.scheduleAtFixedRate({
try {
val state = this.state
if (state is GracefulShutdown) {
val timesToWait = state.lastRunning.clientSettings.gracefulShutdownWaitSeconds / 15
when {
state.counts == 0 -> {
LOGGER.info { "Starting graceful shutdown" }
@ -174,7 +206,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}
} catch (e: Exception) {
LOGGER.warn("Main loop failed", e)
LOGGER.warn(e) { "Main loop failed" }
}
}, 15, 15, TimeUnit.SECONDS)
@ -183,7 +215,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
val state = this.state
if (state is Running) {
val currentBytesSent = statistics.get().bytesSent - lastBytesSent
if (clientSettings.maxMebibytesPerHour != 0L && clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
if (state.clientSettings.maxMebibytesPerHour != 0L && state.clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
LOGGER.info { "Shutting down server as hourly bandwidth limit reached" }
this.state = GracefulShutdown(lastRunning = state)
@ -195,6 +227,18 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
LOGGER.warn(e) { "Graceful shutdown checker failed" }
}
}, 45, 45, TimeUnit.SECONDS)
// Check every minute to see if client settings have changed
executorService.scheduleWithFixedDelay({
try {
val state = this.state
if (state is Running) {
reloadClientSettings()
}
} catch (e: Exception) {
LOGGER.warn(e) { "Reload of ClientSettings failed" }
}
}, 60, 60, TimeUnit.SECONDS)
}
private fun pingControl() {
@ -223,11 +267,11 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
private fun loginAndStartServer() {
this.state as Uninitialized
val state = this.state as Uninitialized
val serverSettings = serverHandler.loginToControl()
?: dieWithError("Failed to get a login response from server - check API secret for validity")
val server = getServer(cache, serverSettings, clientSettings, statistics, isHandled).start()
val server = getServer(cache, serverSettings, state.clientSettings, statistics, isHandled).start()
if (serverSettings.latestBuild > Constants.CLIENT_BUILD) {
LOGGER.warn {
@ -235,7 +279,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}
state = Running(server, serverSettings)
this.state = Running(server, serverSettings, state.clientSettings)
LOGGER.info { "Internal HTTP server was successfully started" }
}
@ -243,7 +287,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
serverHandler.logoutFromControl()
}
private fun stopServer(nextState: State = Uninitialized) {
private fun stopServer(nextState: State) {
val state = this.state.let {
when (it) {
is Running ->
@ -262,6 +306,32 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
this.state = nextState
}
/**
* Starts the WebUI if the ClientSettings demand it.
* Because this method checks if the WebUI is needed,
* it can be safely called before checking yourself
*/
private fun startWebUi() {
val state = this.state
// Grab the client settings if available
val clientSettings = state.let {
when (it) {
is Running ->
it.clientSettings
is Uninitialized ->
it.clientSettings
else ->
null
}
}
// Only start the Web UI if the settings demand it
if (clientSettings?.webSettings != null) {
webUi = getUiServer(clientSettings.webSettings, statistics, statsMap)
webUi!!.start()
}
}
fun shutdown() {
LOGGER.info { "Mangadex@Home Client stopping" }
@ -296,6 +366,131 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
}
/**
* Reloads the client configuration and restarts the
* Web UI and/or the server if needed
*/
private fun reloadClientSettings() {
val state = this.state as Running
LOGGER.info { "Reloading client settings" }
try {
val newSettings = readClientSettings()
if (newSettings == state.clientSettings) {
LOGGER.info { "Client Settings have not changed" }
return
}
// Setting loaded without issue. Figure out
// if there are changes that require a restart
val restartServer = newSettings.clientSecret != state.clientSettings.clientSecret ||
newSettings.clientHostname != state.clientSettings.clientHostname ||
newSettings.clientPort != state.clientSettings.clientPort ||
newSettings.clientExternalPort != state.clientSettings.clientExternalPort ||
newSettings.threads != state.clientSettings.threads ||
newSettings.devSettings?.isDev != state.clientSettings.devSettings?.isDev
val stopWebUi = newSettings.webSettings != state.clientSettings.webSettings ||
newSettings.webSettings?.uiPort != state.clientSettings.webSettings?.uiPort ||
newSettings.webSettings?.uiHostname != state.clientSettings.webSettings?.uiHostname
val startWebUi = (stopWebUi && newSettings.webSettings != null)
// Stop the the WebUI if needed
if (stopWebUi) {
LOGGER.info { "Stopping WebUI to reload ClientSettings" }
webUi?.close()
webUi = null
}
if (restartServer) {
// If we are restarting the server
// We must do it gracefully and set
// the new settings later
LOGGER.info { "Stopping Server to reload ClientSettings" }
this.state = GracefulShutdown(state, nextState = Uninitialized(clientSettings = newSettings), action = {
serverHandler = ServerHandler(newSettings)
LOGGER.info { "Reloaded ClientSettings: $newSettings" }
LOGGER.info { "Starting Server after reloading ClientSettings" }
loginAndStartServer()
// Start the WebUI if we had to stop it
// and still want it
if (startWebUi) {
LOGGER.info { "Starting WebUI after reloading ClientSettings" }
startWebUi()
LOGGER.info { "Started WebUI after reloading ClientSettings" }
}
})
} else {
// If we aren't restarting the server
// We can update the settings now
this.state = state.copy(clientSettings = newSettings)
serverHandler.setClientSettings(newSettings)
LOGGER.info { "Reloaded ClientSettings: $newSettings" }
// Start the WebUI if we had to stop it
// and still want it
if (startWebUi) {
LOGGER.info { "Starting WebUI after reloading ClientSettings" }
startWebUi()
LOGGER.info { "Started WebUI after reloading ClientSettings" }
}
}
} catch (e: UnrecognizedPropertyException) {
LOGGER.warn { "Settings file is invalid: '$e.propertyName' is not a valid setting" }
} catch (e: JsonProcessingException) {
LOGGER.warn { "Settings file is invalid: $e.message" }
} catch (e: IOException) {
LOGGER.warn { "Settings file is could not be found: $e.message" }
} catch (e: ClientSettingsException) {
LOGGER.warn { "Can't reload client settings: $e.message" }
}
}
private fun validateSettings(settings: ClientSettings) {
if (!isSecretValid(settings.clientSecret)) throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
if (settings.clientPort == 0) {
throw ClientSettingsException("Config Error: Invalid port number")
}
if (settings.clientPort in Constants.RESTRICTED_PORTS) {
throw ClientSettingsException("Config Error: Unsafe port number")
}
if (settings.maxCacheSizeInMebibytes < 1024) {
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
}
if (settings.threads < 4) {
throw ClientSettingsException("Config Error: Invalid number of threads, must be >= 4")
}
if (settings.maxMebibytesPerHour < 0) {
throw ClientSettingsException("Config Error: Max bandwidth must be >= 0")
}
if (settings.maxKilobitsPerSecond < 0) {
throw ClientSettingsException("Config Error: Max burst rate must be >= 0")
}
if (settings.gracefulShutdownWaitSeconds < 15) {
throw ClientSettingsException("Config Error: Graceful shutdown wait must be >= 15")
}
if (settings.webSettings != null) {
if (settings.webSettings.uiPort == 0) {
throw ClientSettingsException("Config Error: Invalid UI port number")
}
}
if (settings.experimental != null) {
if (settings.experimental.maxBufferSizeForCacheHit < 0)
throw ClientSettingsException("Config Error: Max cache buffer multiple must be >= 0")
}
}
private fun readClientSettings(): ClientSettings {
return JACKSON.readValue<ClientSettings>(FileReader(clientSettingsFile)).apply(::validateSettings)
}
private fun isSecretValid(clientSecret: String): Boolean {
return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)
}
companion object {
private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper()

View file

@ -40,7 +40,7 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
)
class ServerHandler(private val settings: ClientSettings) {
class ServerHandler(private var settings: ClientSettings) {
private val client = ApacheClient()
fun logoutFromControl(): Boolean {
@ -109,6 +109,10 @@ class ServerHandler(private val settings: ClientSettings) {
SERVER_ADDRESS_DEV
}
fun setClientSettings(clientSettings: ClientSettings) {
this.settings = clientSettings
}
companion object {
private val LOGGER = LoggerFactory.getLogger(ServerHandler::class.java)
private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens()

View file

@ -81,8 +81,6 @@ class ImageServer(
?.times(8 * 1024)
fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler {
val sodium = LazySodiumJava(SodiumJava())
return baseHandler().then { request ->
val chapterHash = Path.of("chapterHash")(request)
val fileName = Path.of("fileName")(request)
@ -100,10 +98,14 @@ class ImageServer(
if (tokenized || serverSettings.forceTokens) {
val tokenArr = Base64.getUrlDecoder().decode(Path.of("token")(request))
if (tokenArr.size < 24) {
LOGGER.info { "Request for $sanitizedUri rejected for invalid token" }
return@then Response(Status.FORBIDDEN)
}
val token = try {
JACKSON.readValue<Token>(
try {
sodium.cryptoBoxOpenEasyAfterNm(
SODIUM.cryptoBoxOpenEasyAfterNm(
tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24), serverSettings.tokenKey
)
} catch (_: SodiumException) {
@ -145,12 +147,12 @@ class ImageServer(
}
}
if (snapshot != null && imageDatum != null) {
if (snapshot != null && imageDatum != null && imageDatum.contentType.isImageMimetype()) {
request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
} else {
if (snapshot != null) {
snapshot.close()
LOGGER.warn { "Removing cache file for $sanitizedUri without corresponding DB entry" }
LOGGER.warn { "Removing broken cache file for $sanitizedUri" }
cache.removeUnsafe(imageId.toCacheId())
}
@ -224,12 +226,18 @@ class ImageServer(
return Response(mdResponse.status)
}
LOGGER.trace { "Upstream query for $sanitizedUri succeeded" }
val contentType = mdResponse.header("Content-Type")!!
val contentLength = mdResponse.header("Content-Length")
val lastModified = mdResponse.header("Last-Modified")
if (!contentType.isImageMimetype()) {
LOGGER.trace { "Upstream query for $sanitizedUri returned bad mimetype $contentType" }
mdResponse.close()
return Response(Status.INTERNAL_SERVER_ERROR)
}
LOGGER.trace { "Upstream query for $sanitizedUri succeeded" }
val editor = cache.editUnsafe(imageId.toCacheId())
// A null editor means that this file is being written to
@ -297,6 +305,7 @@ class ImageServer(
.header("X-Cache", if (cached) "HIT" else "MISS")
companion object {
private val SODIUM = LazySodiumJava(SodiumJava())
private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@ -335,3 +344,5 @@ private fun printHexString(bytes: ByteArray): String {
}
return sb.toString()
}
private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/")