499 lines
20 KiB
Kotlin
499 lines
20 KiB
Kotlin
/*
|
|
Mangadex@Home
|
|
Copyright (c) 2020, MangaDex Network
|
|
This file is part of MangaDex@Home.
|
|
|
|
MangaDex@Home is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
MangaDex@Home is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
/* ktlint-disable no-wildcard-imports */
|
|
package mdnet.base
|
|
|
|
import ch.qos.logback.classic.LoggerContext
|
|
import com.fasterxml.jackson.core.JsonProcessingException
|
|
import com.fasterxml.jackson.databind.ObjectMapper
|
|
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
|
|
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
|
import com.fasterxml.jackson.module.kotlin.readValue
|
|
import java.io.File
|
|
import java.io.FileReader
|
|
import java.io.FileWriter
|
|
import java.io.IOException
|
|
import java.time.Instant
|
|
import java.util.*
|
|
import java.util.concurrent.CountDownLatch
|
|
import java.util.concurrent.Executors
|
|
import java.util.concurrent.TimeUnit
|
|
import java.util.concurrent.atomic.AtomicBoolean
|
|
import java.util.concurrent.atomic.AtomicReference
|
|
import java.util.regex.Pattern
|
|
import mdnet.base.Main.dieWithError
|
|
import mdnet.base.data.Statistics
|
|
import mdnet.base.server.getServer
|
|
import mdnet.base.server.getUiServer
|
|
import mdnet.base.settings.ClientSettings
|
|
import mdnet.base.settings.ServerSettings
|
|
import mdnet.cache.DiskLruCache
|
|
import mdnet.cache.HeaderMismatchException
|
|
import org.http4k.server.Http4kServer
|
|
import org.slf4j.LoggerFactory
|
|
|
|
private const val CLIENT_KEY_LENGTH = 52
|
|
|
|
// Exception class to handle when Client Settings have invalid values
|
|
class ClientSettingsException(message: String) : Exception(message)
|
|
|
|
sealed class State
|
|
// server is not running
|
|
data class Uninitialized(val clientSettings: ClientSettings) : State()
|
|
// server has shut down
|
|
object Shutdown : State()
|
|
// server is in the process of shutting down
|
|
data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized(lastRunning.clientSettings), val action: () -> Unit = {}) : State()
|
|
// server is currently running
|
|
data class Running(val server: Http4kServer, val settings: ServerSettings, val clientSettings: ClientSettings) : State()
|
|
// clientSettings must only be accessed from the thread on the executorService
|
|
class MangaDexClient(private val clientSettingsFile: String) {
|
|
// this must remain singlethreaded because of how the state mechanism works
|
|
private val executorService = Executors.newSingleThreadScheduledExecutor()
|
|
// state must only be accessed from the thread on the executorService
|
|
private var state: State
|
|
|
|
private var serverHandler: ServerHandler
|
|
private val statsMap: MutableMap<Instant, Statistics> = Collections
|
|
.synchronizedMap(object : LinkedHashMap<Instant, Statistics>(240) {
|
|
override fun removeEldestEntry(eldest: Map.Entry<Instant, Statistics>): Boolean {
|
|
return this.size > 240
|
|
}
|
|
})
|
|
private val statistics: AtomicReference<Statistics> = AtomicReference(
|
|
Statistics()
|
|
)
|
|
private val isHandled: AtomicBoolean = AtomicBoolean(false)
|
|
private var webUi: Http4kServer? = null
|
|
private val cache: DiskLruCache
|
|
|
|
init {
|
|
// Read ClientSettings
|
|
val clientSettings = try {
|
|
readClientSettings()
|
|
} catch (e: UnrecognizedPropertyException) {
|
|
dieWithError("'${e.propertyName}' is not a valid setting")
|
|
} catch (e: JsonProcessingException) {
|
|
dieWithError(e)
|
|
} catch (ignored: IOException) {
|
|
ClientSettings().also {
|
|
LOGGER.warn { "Settings file $clientSettingsFile not found, generating file" }
|
|
try {
|
|
FileWriter(clientSettingsFile).use { writer -> JACKSON.writeValue(writer, it) }
|
|
} catch (e: IOException) {
|
|
dieWithError(e)
|
|
}
|
|
}
|
|
} catch (e: ClientSettingsException) {
|
|
dieWithError(e)
|
|
}
|
|
|
|
// Initialize things that depend on Client Settings
|
|
LOGGER.info { "Client settings loaded: $clientSettings" }
|
|
state = Uninitialized(clientSettings)
|
|
serverHandler = ServerHandler(clientSettings)
|
|
|
|
// Initialize everything else
|
|
try {
|
|
cache = DiskLruCache.open(
|
|
File("cache"), 1, 1,
|
|
clientSettings.maxCacheSizeInMebibytes * 1024 * 1024 /* MiB to bytes */
|
|
)
|
|
cache.get("statistics")?.use {
|
|
statistics.set(JACKSON.readValue<Statistics>(it.getInputStream(0)))
|
|
}
|
|
} catch (e: HeaderMismatchException) {
|
|
LOGGER.warn { "Cache version may be outdated - remove if necessary" }
|
|
dieWithError(e)
|
|
} catch (e: IOException) {
|
|
LOGGER.warn { "Cache may be corrupt - remove if necessary" }
|
|
dieWithError(e)
|
|
}
|
|
}
|
|
|
|
fun runLoop() {
|
|
loginAndStartServer()
|
|
statsMap[Instant.now()] = statistics.get()
|
|
startWebUi()
|
|
LOGGER.info { "Mangadex@Home Client initialized. Starting normal operation." }
|
|
|
|
executorService.scheduleAtFixedRate({
|
|
try {
|
|
if (state is Running || state is GracefulShutdown || state is Uninitialized) {
|
|
statistics.updateAndGet {
|
|
it.copy(bytesOnDisk = cache.size())
|
|
}
|
|
statsMap[Instant.now()] = statistics.get()
|
|
val editor = cache.edit("statistics")
|
|
if (editor != null) {
|
|
JACKSON.writeValue(editor.newOutputStream(0), statistics.get())
|
|
editor.commit()
|
|
}
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Statistics update failed" }
|
|
}
|
|
}, 15, 15, TimeUnit.SECONDS)
|
|
|
|
var lastBytesSent = statistics.get().bytesSent
|
|
executorService.scheduleAtFixedRate({
|
|
try {
|
|
lastBytesSent = statistics.get().bytesSent
|
|
|
|
val state = this.state
|
|
if (state is GracefulShutdown) {
|
|
LOGGER.info { "Aborting graceful shutdown started due to hourly bandwidth limit" }
|
|
|
|
this.state = state.lastRunning
|
|
}
|
|
if (state is Uninitialized) {
|
|
LOGGER.info { "Restarting server stopped due to hourly bandwidth limit" }
|
|
|
|
loginAndStartServer()
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Hourly bandwidth check failed" }
|
|
}
|
|
}, 1, 1, TimeUnit.HOURS)
|
|
|
|
executorService.scheduleAtFixedRate({
|
|
try {
|
|
val state = this.state
|
|
if (state is GracefulShutdown) {
|
|
val timesToWait = state.lastRunning.clientSettings.gracefulShutdownWaitSeconds / 15
|
|
when {
|
|
state.counts == 0 -> {
|
|
LOGGER.info { "Starting graceful shutdown" }
|
|
|
|
logout()
|
|
isHandled.set(false)
|
|
this.state = state.copy(counts = state.counts + 1)
|
|
}
|
|
state.counts == timesToWait || !isHandled.get() -> {
|
|
if (!isHandled.get()) {
|
|
LOGGER.info { "No requests received, shutting down" }
|
|
} else {
|
|
LOGGER.info { "Max tries attempted (${state.counts} out of $timesToWait), shutting down" }
|
|
}
|
|
|
|
stopServer(state.nextState)
|
|
state.action()
|
|
}
|
|
else -> {
|
|
LOGGER.info {
|
|
"Waiting another 15 seconds for graceful shutdown (${state.counts} out of $timesToWait)"
|
|
}
|
|
|
|
isHandled.set(false)
|
|
this.state = state.copy(counts = state.counts + 1)
|
|
}
|
|
}
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Main loop failed" }
|
|
}
|
|
}, 15, 15, TimeUnit.SECONDS)
|
|
|
|
executorService.scheduleWithFixedDelay({
|
|
try {
|
|
val state = this.state
|
|
if (state is Running) {
|
|
val currentBytesSent = statistics.get().bytesSent - lastBytesSent
|
|
if (state.clientSettings.maxMebibytesPerHour != 0L && state.clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
|
|
LOGGER.info { "Shutting down server as hourly bandwidth limit reached" }
|
|
|
|
this.state = GracefulShutdown(lastRunning = state)
|
|
} else {
|
|
pingControl()
|
|
}
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Graceful shutdown checker failed" }
|
|
}
|
|
}, 45, 45, TimeUnit.SECONDS)
|
|
|
|
// Check every minute to see if client settings have changed
|
|
executorService.scheduleWithFixedDelay({
|
|
try {
|
|
val state = this.state
|
|
if (state is Running) {
|
|
reloadClientSettings()
|
|
}
|
|
} catch (e: Exception) {
|
|
LOGGER.warn(e) { "Reload of ClientSettings failed" }
|
|
}
|
|
}, 60, 60, TimeUnit.SECONDS)
|
|
}
|
|
|
|
private fun pingControl() {
|
|
val state = this.state as Running
|
|
|
|
val newSettings = serverHandler.pingControl(state.settings)
|
|
if (newSettings != null) {
|
|
LOGGER.info { "Server settings received: $newSettings" }
|
|
|
|
if (newSettings.latestBuild > Constants.CLIENT_BUILD) {
|
|
LOGGER.warn {
|
|
"Outdated build detected! Latest: ${newSettings.latestBuild}, Current: ${Constants.CLIENT_BUILD}"
|
|
}
|
|
}
|
|
if (newSettings.tls != null || newSettings.imageServer != state.settings.imageServer) {
|
|
// certificates or upstream url must have changed, restart webserver
|
|
LOGGER.info { "Doing internal restart of HTTP server to refresh certs/upstream URL" }
|
|
|
|
this.state = GracefulShutdown(lastRunning = state) {
|
|
loginAndStartServer()
|
|
}
|
|
}
|
|
} else {
|
|
LOGGER.info { "Server ping failed - ignoring" }
|
|
}
|
|
}
|
|
|
|
private fun loginAndStartServer() {
|
|
val state = this.state as Uninitialized
|
|
|
|
val serverSettings = serverHandler.loginToControl()
|
|
?: dieWithError("Failed to get a login response from server - check API secret for validity")
|
|
val server = getServer(cache, serverSettings, state.clientSettings, statistics, isHandled).start()
|
|
|
|
if (serverSettings.latestBuild > Constants.CLIENT_BUILD) {
|
|
LOGGER.warn {
|
|
"Outdated build detected! Latest: ${serverSettings.latestBuild}, Current: ${Constants.CLIENT_BUILD}"
|
|
}
|
|
}
|
|
|
|
this.state = Running(server, serverSettings, state.clientSettings)
|
|
LOGGER.info { "Internal HTTP server was successfully started" }
|
|
}
|
|
|
|
private fun logout() {
|
|
serverHandler.logoutFromControl()
|
|
}
|
|
|
|
private fun stopServer(nextState: State) {
|
|
val state = this.state.let {
|
|
when (it) {
|
|
is Running ->
|
|
it
|
|
is GracefulShutdown ->
|
|
it.lastRunning
|
|
else ->
|
|
throw AssertionError()
|
|
}
|
|
}
|
|
|
|
LOGGER.info { "Shutting down HTTP server" }
|
|
state.server.stop()
|
|
LOGGER.info { "Internal HTTP server has shut down" }
|
|
|
|
this.state = nextState
|
|
}
|
|
|
|
/**
|
|
* Starts the WebUI if the ClientSettings demand it.
|
|
* Because this method checks if the WebUI is needed,
|
|
* it can be safely called before checking yourself
|
|
*/
|
|
private fun startWebUi() {
|
|
val state = this.state
|
|
// Grab the client settings if available
|
|
val clientSettings = state.let {
|
|
when (it) {
|
|
is Running ->
|
|
it.clientSettings
|
|
is Uninitialized ->
|
|
it.clientSettings
|
|
else ->
|
|
null
|
|
}
|
|
}
|
|
|
|
// Only start the Web UI if the settings demand it
|
|
if (clientSettings?.webSettings != null) {
|
|
webUi = getUiServer(clientSettings.webSettings, statistics, statsMap)
|
|
webUi!!.start()
|
|
}
|
|
}
|
|
|
|
fun shutdown() {
|
|
LOGGER.info { "Mangadex@Home Client stopping" }
|
|
|
|
val latch = CountDownLatch(1)
|
|
executorService.schedule({
|
|
val state = this.state
|
|
if (state is Running) {
|
|
this.state = GracefulShutdown(state, nextState = Shutdown) {
|
|
latch.countDown()
|
|
}
|
|
} else if (state is GracefulShutdown) {
|
|
this.state = state.copy(nextState = Shutdown) {
|
|
latch.countDown()
|
|
}
|
|
} else if (state is Uninitialized || state is Shutdown) {
|
|
this.state = Shutdown
|
|
latch.countDown()
|
|
}
|
|
}, 0, TimeUnit.SECONDS)
|
|
latch.await()
|
|
|
|
webUi?.close()
|
|
try {
|
|
cache.close()
|
|
} catch (e: IOException) {
|
|
LOGGER.error(e) { "Cache failed to close" }
|
|
}
|
|
|
|
executorService.shutdown()
|
|
LOGGER.info { "Mangadex@Home Client stopped" }
|
|
|
|
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
|
|
}
|
|
|
|
/**
|
|
* Reloads the client configuration and restarts the
|
|
* Web UI and/or the server if needed
|
|
*/
|
|
private fun reloadClientSettings() {
|
|
val state = this.state as Running
|
|
LOGGER.info { "Reloading client settings" }
|
|
try {
|
|
val newSettings = readClientSettings()
|
|
|
|
if (newSettings == state.clientSettings) {
|
|
LOGGER.info { "Client Settings have not changed" }
|
|
return
|
|
}
|
|
|
|
// Setting loaded without issue. Figure out
|
|
// if there are changes that require a restart
|
|
val restartServer = newSettings.clientSecret != state.clientSettings.clientSecret ||
|
|
newSettings.clientHostname != state.clientSettings.clientHostname ||
|
|
newSettings.clientPort != state.clientSettings.clientPort ||
|
|
newSettings.clientExternalPort != state.clientSettings.clientExternalPort ||
|
|
newSettings.threads != state.clientSettings.threads ||
|
|
newSettings.devSettings?.isDev != state.clientSettings.devSettings?.isDev
|
|
val stopWebUi = newSettings.webSettings != state.clientSettings.webSettings ||
|
|
newSettings.webSettings?.uiPort != state.clientSettings.webSettings?.uiPort ||
|
|
newSettings.webSettings?.uiHostname != state.clientSettings.webSettings?.uiHostname
|
|
val startWebUi = (stopWebUi && newSettings.webSettings != null)
|
|
|
|
// Stop the the WebUI if needed
|
|
if (stopWebUi) {
|
|
LOGGER.info { "Stopping WebUI to reload ClientSettings" }
|
|
webUi?.close()
|
|
webUi = null
|
|
}
|
|
|
|
if (restartServer) {
|
|
// If we are restarting the server
|
|
// We must do it gracefully and set
|
|
// the new settings later
|
|
LOGGER.info { "Stopping Server to reload ClientSettings" }
|
|
|
|
this.state = GracefulShutdown(state, nextState = Uninitialized(clientSettings = newSettings), action = {
|
|
serverHandler = ServerHandler(newSettings)
|
|
LOGGER.info { "Reloaded ClientSettings: $newSettings" }
|
|
|
|
LOGGER.info { "Starting Server after reloading ClientSettings" }
|
|
loginAndStartServer()
|
|
|
|
// Start the WebUI if we had to stop it
|
|
// and still want it
|
|
if (startWebUi) {
|
|
LOGGER.info { "Starting WebUI after reloading ClientSettings" }
|
|
startWebUi()
|
|
LOGGER.info { "Started WebUI after reloading ClientSettings" }
|
|
}
|
|
})
|
|
} else {
|
|
// If we aren't restarting the server
|
|
// We can update the settings now
|
|
this.state = state.copy(clientSettings = newSettings)
|
|
serverHandler.setClientSettings(newSettings)
|
|
LOGGER.info { "Reloaded ClientSettings: $newSettings" }
|
|
|
|
// Start the WebUI if we had to stop it
|
|
// and still want it
|
|
if (startWebUi) {
|
|
LOGGER.info { "Starting WebUI after reloading ClientSettings" }
|
|
startWebUi()
|
|
LOGGER.info { "Started WebUI after reloading ClientSettings" }
|
|
}
|
|
}
|
|
} catch (e: UnrecognizedPropertyException) {
|
|
LOGGER.warn { "Settings file is invalid: '$e.propertyName' is not a valid setting" }
|
|
} catch (e: JsonProcessingException) {
|
|
LOGGER.warn { "Settings file is invalid: $e.message" }
|
|
} catch (e: IOException) {
|
|
LOGGER.warn { "Settings file is could not be found: $e.message" }
|
|
} catch (e: ClientSettingsException) {
|
|
LOGGER.warn { "Can't reload client settings: $e.message" }
|
|
}
|
|
}
|
|
|
|
private fun validateSettings(settings: ClientSettings) {
|
|
if (!isSecretValid(settings.clientSecret)) throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
|
|
if (settings.clientPort == 0) {
|
|
throw ClientSettingsException("Config Error: Invalid port number")
|
|
}
|
|
if (settings.clientPort in Constants.RESTRICTED_PORTS) {
|
|
throw ClientSettingsException("Config Error: Unsafe port number")
|
|
}
|
|
if (settings.maxCacheSizeInMebibytes < 1024) {
|
|
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
|
|
}
|
|
if (settings.threads < 4) {
|
|
throw ClientSettingsException("Config Error: Invalid number of threads, must be >= 4")
|
|
}
|
|
if (settings.maxMebibytesPerHour < 0) {
|
|
throw ClientSettingsException("Config Error: Max bandwidth must be >= 0")
|
|
}
|
|
if (settings.maxKilobitsPerSecond < 0) {
|
|
throw ClientSettingsException("Config Error: Max burst rate must be >= 0")
|
|
}
|
|
if (settings.gracefulShutdownWaitSeconds < 15) {
|
|
throw ClientSettingsException("Config Error: Graceful shutdown wait must be >= 15")
|
|
}
|
|
if (settings.webSettings != null) {
|
|
if (settings.webSettings.uiPort == 0) {
|
|
throw ClientSettingsException("Config Error: Invalid UI port number")
|
|
}
|
|
}
|
|
if (settings.experimental != null) {
|
|
if (settings.experimental.maxBufferSizeForCacheHit < 0)
|
|
throw ClientSettingsException("Config Error: Max cache buffer multiple must be >= 0")
|
|
}
|
|
|
|
}
|
|
|
|
private fun readClientSettings(): ClientSettings {
|
|
return JACKSON.readValue<ClientSettings>(FileReader(clientSettingsFile)).apply(::validateSettings)
|
|
}
|
|
|
|
private fun isSecretValid(clientSecret: String): Boolean {
|
|
return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)
|
|
}
|
|
|
|
companion object {
|
|
private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java)
|
|
private val JACKSON: ObjectMapper = jacksonObjectMapper()
|
|
}
|
|
}
|