/* Mangadex@Home Copyright (c) 2020, MangaDex Network This file is part of MangaDex@Home. MangaDex@Home is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. MangaDex@Home is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this MangaDex@Home. If not, see . */ /* ktlint-disable no-wildcard-imports */ package mdnet.base import ch.qos.logback.classic.LoggerContext import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import java.io.File import java.io.FileReader import java.io.FileWriter import java.io.IOException import java.time.Instant import java.util.* import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference import java.util.regex.Pattern import mdnet.base.Main.dieWithError import mdnet.base.data.Statistics import mdnet.base.server.getServer import mdnet.base.server.getUiServer import mdnet.base.settings.ClientSettings import mdnet.base.settings.ServerSettings import mdnet.cache.DiskLruCache import mdnet.cache.HeaderMismatchException import org.http4k.server.Http4kServer import org.slf4j.LoggerFactory private const val CLIENT_KEY_LENGTH = 52 // Exception class to handle when Client Settings have invalid values class ClientSettingsException(message: String) : Exception(message) sealed class State // server is not running data class Uninitialized(val clientSettings: ClientSettings) : State() // server has shut down object Shutdown : State() // server is in the process of shutting down data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized(lastRunning.clientSettings), val action: () -> Unit = {}) : State() // server is currently running data class Running(val server: Http4kServer, val settings: ServerSettings, val clientSettings: ClientSettings) : State() // clientSettings must only be accessed from the thread on the executorService class MangaDexClient(private val clientSettingsFile: String) { // this must remain singlethreaded because of how the state mechanism works private val executorService = Executors.newSingleThreadScheduledExecutor() // state must only be accessed from the thread on the executorService private var state: State private var serverHandler: ServerHandler private val statsMap: MutableMap = Collections .synchronizedMap(object : LinkedHashMap(240) { override fun removeEldestEntry(eldest: Map.Entry): Boolean { return this.size > 240 } }) private val statistics: AtomicReference = AtomicReference( Statistics() ) private val isHandled: AtomicBoolean = AtomicBoolean(false) private var webUi: Http4kServer? = null private val cache: DiskLruCache init { // Read ClientSettings val clientSettings = try { readClientSettings() } catch (e: UnrecognizedPropertyException) { dieWithError("'${e.propertyName}' is not a valid setting") } catch (e: JsonProcessingException) { dieWithError(e) } catch (ignored: IOException) { ClientSettings().also { LOGGER.warn { "Settings file $clientSettingsFile not found, generating file" } try { FileWriter(clientSettingsFile).use { writer -> JACKSON.writeValue(writer, it) } } catch (e: IOException) { dieWithError(e) } } } catch (e: ClientSettingsException) { dieWithError(e) } // Initialize things that depend on Client Settings LOGGER.info { "Client settings loaded: $clientSettings" } state = Uninitialized(clientSettings) serverHandler = ServerHandler(clientSettings) // Initialize everything else try { cache = DiskLruCache.open( File("cache"), 1, 1, clientSettings.maxCacheSizeInMebibytes * 1024 * 1024 /* MiB to bytes */ ) cache.get("statistics")?.use { statistics.set(JACKSON.readValue(it.getInputStream(0))) } } catch (e: HeaderMismatchException) { LOGGER.warn { "Cache version may be outdated - remove if necessary" } dieWithError(e) } catch (e: IOException) { LOGGER.warn { "Cache may be corrupt - remove if necessary" } dieWithError(e) } } fun runLoop() { loginAndStartServer() statsMap[Instant.now()] = statistics.get() startWebUi() LOGGER.info { "Mangadex@Home Client initialized. Starting normal operation." } executorService.scheduleAtFixedRate({ try { if (state is Running || state is GracefulShutdown || state is Uninitialized) { statistics.updateAndGet { it.copy(bytesOnDisk = cache.size()) } statsMap[Instant.now()] = statistics.get() val editor = cache.edit("statistics") if (editor != null) { JACKSON.writeValue(editor.newOutputStream(0), statistics.get()) editor.commit() } } } catch (e: Exception) { LOGGER.warn(e) { "Statistics update failed" } } }, 15, 15, TimeUnit.SECONDS) var lastBytesSent = statistics.get().bytesSent executorService.scheduleAtFixedRate({ try { lastBytesSent = statistics.get().bytesSent val state = this.state if (state is GracefulShutdown) { LOGGER.info { "Aborting graceful shutdown started due to hourly bandwidth limit" } this.state = state.lastRunning } if (state is Uninitialized) { LOGGER.info { "Restarting server stopped due to hourly bandwidth limit" } loginAndStartServer() } } catch (e: Exception) { LOGGER.warn(e) { "Hourly bandwidth check failed" } } }, 1, 1, TimeUnit.HOURS) executorService.scheduleAtFixedRate({ try { val state = this.state if (state is GracefulShutdown) { val timesToWait = state.lastRunning.clientSettings.gracefulShutdownWaitSeconds / 15 when { state.counts == 0 -> { LOGGER.info { "Starting graceful shutdown" } logout() isHandled.set(false) this.state = state.copy(counts = state.counts + 1) } state.counts == timesToWait || !isHandled.get() -> { if (!isHandled.get()) { LOGGER.info { "No requests received, shutting down" } } else { LOGGER.info { "Max tries attempted (${state.counts} out of $timesToWait), shutting down" } } stopServer(state.nextState) state.action() } else -> { LOGGER.info { "Waiting another 15 seconds for graceful shutdown (${state.counts} out of $timesToWait)" } isHandled.set(false) this.state = state.copy(counts = state.counts + 1) } } } } catch (e: Exception) { LOGGER.warn(e) { "Main loop failed" } } }, 15, 15, TimeUnit.SECONDS) executorService.scheduleWithFixedDelay({ try { val state = this.state if (state is Running) { val currentBytesSent = statistics.get().bytesSent - lastBytesSent if (state.clientSettings.maxMebibytesPerHour != 0L && state.clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) { LOGGER.info { "Shutting down server as hourly bandwidth limit reached" } this.state = GracefulShutdown(lastRunning = state) } else { pingControl() } } } catch (e: Exception) { LOGGER.warn(e) { "Graceful shutdown checker failed" } } }, 45, 45, TimeUnit.SECONDS) // Check every minute to see if client settings have changed executorService.scheduleWithFixedDelay({ try { val state = this.state if (state is Running) { reloadClientSettings() } } catch (e: Exception) { LOGGER.warn(e) { "Reload of ClientSettings failed" } } }, 60, 60, TimeUnit.SECONDS) } private fun pingControl() { val state = this.state as Running val newSettings = serverHandler.pingControl(state.settings) if (newSettings != null) { LOGGER.info { "Server settings received: $newSettings" } if (newSettings.latestBuild > Constants.CLIENT_BUILD) { LOGGER.warn { "Outdated build detected! Latest: ${newSettings.latestBuild}, Current: ${Constants.CLIENT_BUILD}" } } if (newSettings.tls != null || newSettings.imageServer != state.settings.imageServer) { // certificates or upstream url must have changed, restart webserver LOGGER.info { "Doing internal restart of HTTP server to refresh certs/upstream URL" } this.state = GracefulShutdown(lastRunning = state) { loginAndStartServer() } } } else { LOGGER.info { "Server ping failed - ignoring" } } } private fun loginAndStartServer() { val state = this.state as Uninitialized val serverSettings = serverHandler.loginToControl() ?: dieWithError("Failed to get a login response from server - check API secret for validity") val server = getServer(cache, serverSettings, state.clientSettings, statistics, isHandled).start() if (serverSettings.latestBuild > Constants.CLIENT_BUILD) { LOGGER.warn { "Outdated build detected! Latest: ${serverSettings.latestBuild}, Current: ${Constants.CLIENT_BUILD}" } } this.state = Running(server, serverSettings, state.clientSettings) LOGGER.info { "Internal HTTP server was successfully started" } } private fun logout() { serverHandler.logoutFromControl() } private fun stopServer(nextState: State) { val state = this.state.let { when (it) { is Running -> it is GracefulShutdown -> it.lastRunning else -> throw AssertionError() } } LOGGER.info { "Shutting down HTTP server" } state.server.stop() LOGGER.info { "Internal HTTP server has shut down" } this.state = nextState } /** * Starts the WebUI if the ClientSettings demand it. * Because this method checks if the WebUI is needed, * it can be safely called before checking yourself */ private fun startWebUi() { val state = this.state // Grab the client settings if available val clientSettings = state.let { when (it) { is Running -> it.clientSettings is Uninitialized -> it.clientSettings else -> null } } // Only start the Web UI if the settings demand it if (clientSettings?.webSettings != null) { webUi = getUiServer(clientSettings.webSettings, statistics, statsMap) webUi!!.start() } } fun shutdown() { LOGGER.info { "Mangadex@Home Client stopping" } val latch = CountDownLatch(1) executorService.schedule({ val state = this.state if (state is Running) { this.state = GracefulShutdown(state, nextState = Shutdown) { latch.countDown() } } else if (state is GracefulShutdown) { this.state = state.copy(nextState = Shutdown) { latch.countDown() } } else if (state is Uninitialized || state is Shutdown) { this.state = Shutdown latch.countDown() } }, 0, TimeUnit.SECONDS) latch.await() webUi?.close() try { cache.close() } catch (e: IOException) { LOGGER.error(e) { "Cache failed to close" } } executorService.shutdown() LOGGER.info { "Mangadex@Home Client stopped" } (LoggerFactory.getILoggerFactory() as LoggerContext).stop() } /** * Reloads the client configuration and restarts the * Web UI and/or the server if needed */ private fun reloadClientSettings() { val state = this.state as Running LOGGER.info { "Reloading client settings" } try { val newSettings = readClientSettings() if (newSettings == state.clientSettings) { LOGGER.info { "Client Settings have not changed" } return } // Setting loaded without issue. Figure out // if there are changes that require a restart val restartServer = newSettings.clientSecret != state.clientSettings.clientSecret || newSettings.clientHostname != state.clientSettings.clientHostname || newSettings.clientPort != state.clientSettings.clientPort || newSettings.clientExternalPort != state.clientSettings.clientExternalPort || newSettings.threads != state.clientSettings.threads || newSettings.devSettings?.isDev != state.clientSettings.devSettings?.isDev val stopWebUi = newSettings.webSettings != state.clientSettings.webSettings || newSettings.webSettings?.uiPort != state.clientSettings.webSettings?.uiPort || newSettings.webSettings?.uiHostname != state.clientSettings.webSettings?.uiHostname val startWebUi = (stopWebUi && newSettings.webSettings != null) // Stop the the WebUI if needed if (stopWebUi) { LOGGER.info { "Stopping WebUI to reload ClientSettings" } webUi?.close() webUi = null } if (restartServer) { // If we are restarting the server // We must do it gracefully and set // the new settings later LOGGER.info { "Stopping Server to reload ClientSettings" } this.state = GracefulShutdown(state, nextState = Uninitialized(clientSettings = newSettings), action = { serverHandler = ServerHandler(newSettings) LOGGER.info { "Reloaded ClientSettings: $newSettings" } LOGGER.info { "Starting Server after reloading ClientSettings" } loginAndStartServer() // Start the WebUI if we had to stop it // and still want it if (startWebUi) { LOGGER.info { "Starting WebUI after reloading ClientSettings" } startWebUi() LOGGER.info { "Started WebUI after reloading ClientSettings" } } }) } else { // If we aren't restarting the server // We can update the settings now this.state = state.copy(clientSettings = newSettings) serverHandler.setClientSettings(newSettings) LOGGER.info { "Reloaded ClientSettings: $newSettings" } // Start the WebUI if we had to stop it // and still want it if (startWebUi) { LOGGER.info { "Starting WebUI after reloading ClientSettings" } startWebUi() LOGGER.info { "Started WebUI after reloading ClientSettings" } } } } catch (e: UnrecognizedPropertyException) { LOGGER.warn { "Settings file is invalid: '$e.propertyName' is not a valid setting" } } catch (e: JsonProcessingException) { LOGGER.warn { "Settings file is invalid: $e.message" } } catch (e: IOException) { LOGGER.warn { "Settings file is could not be found: $e.message" } } catch (e: ClientSettingsException) { LOGGER.warn { "Can't reload client settings: $e.message" } } } private fun validateSettings(settings: ClientSettings) { if (!isSecretValid(settings.clientSecret)) throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters") if (settings.clientPort == 0) { throw ClientSettingsException("Config Error: Invalid port number") } if (settings.clientPort in Constants.RESTRICTED_PORTS) { throw ClientSettingsException("Config Error: Unsafe port number") } if (settings.maxCacheSizeInMebibytes < 1024) { throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)") } if (settings.threads < 4) { throw ClientSettingsException("Config Error: Invalid number of threads, must be >= 4") } if (settings.maxMebibytesPerHour < 0) { throw ClientSettingsException("Config Error: Max bandwidth must be >= 0") } if (settings.maxKilobitsPerSecond < 0) { throw ClientSettingsException("Config Error: Max burst rate must be >= 0") } if (settings.gracefulShutdownWaitSeconds < 15) { throw ClientSettingsException("Config Error: Graceful shutdown wait must be >= 15") } if (settings.webSettings != null) { if (settings.webSettings.uiPort == 0) { throw ClientSettingsException("Config Error: Invalid UI port number") } } if (settings.experimental != null) { if (settings.experimental.maxBufferSizeForCacheHit < 0) throw ClientSettingsException("Config Error: Max cache buffer multiple must be >= 0") } } private fun readClientSettings(): ClientSettings { return JACKSON.readValue(FileReader(clientSettingsFile)).apply(::validateSettings) } private fun isSecretValid(clientSecret: String): Boolean { return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret) } companion object { private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java) private val JACKSON: ObjectMapper = jacksonObjectMapper() } }