1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Fix a bunch of various nits

This commit is contained in:
carbotaniuman 2021-01-24 12:05:05 -06:00
parent dc53eea42f
commit 5ee6dd23cd
11 changed files with 122 additions and 37 deletions

2
.gitignore vendored
View file

@ -107,7 +107,7 @@ log/**
images/** images/**
*.db *.db
*settings.yaml settings.yaml
/cache /cache
docker/data docker/data

View file

@ -14,11 +14,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ### Removed
### Fixed ### Fixed
- [2021-01-23] Fix sample docker-compose.yml [@_tde9]
### Security ### Security
## [2.0.0-rc1] - 2020-01-23 ## [2.0.0-rc2] - 2021-01-24
### Added
- [2021-01-24] Epoll and IOUring HTTP transports [@carbotaniuman].
### Changed
- [2021-01-23] Log files are now UTF-8 [@_tde9]
### Fixed
- [2021-01-23] Fix sample docker-compose.yml [@_tde9]
- [2021-01-24] Fix another race condition [@carbotaniuman]
- [2021-01-23] Fix validating small (<1024MiB) caches [@_tde9]
## [2.0.0-rc1] - 2021-01-23
This release contains many breaking changes! Of note are the changes to the cache folders, database location, and settings format. This release contains many breaking changes! Of note are the changes to the cache folders, database location, and settings format.
### Added ### Added
- [2021-01-23] Added `external_max_kilobits_per_second` config option [@carbotaniuman]. - [2021-01-23] Added `external_max_kilobits_per_second` config option [@carbotaniuman].
@ -280,7 +291,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed ### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc1...HEAD [Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc2...HEAD
[2.0.0-rc2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc1...2.0.0-rc2
[2.0.0-rc1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.4...2.0.0-rc1 [2.0.0-rc1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.4...2.0.0-rc1
[1.2.4]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.3...1.2.4 [1.2.4]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.3...1.2.4
[1.2.3]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.2...1.2.3 [1.2.3]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.2...1.2.3

View file

@ -39,6 +39,8 @@ dependencies {
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version" testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final" runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"

View file

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration import java.time.Duration
object Constants { object Constants {
const val CLIENT_BUILD = 21 const val CLIENT_BUILD = 22
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

View file

@ -188,9 +188,9 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo
} }
private fun validateSettings(settings: ClientSettings) { private fun validateSettings(settings: ClientSettings) {
// if (settings.maxCacheSizeInMebibytes < 1024) { if (settings.maxCacheSizeInMebibytes < 1024) {
// throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)") throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
// } }
fun isSecretValid(clientSecret: String): Boolean { fun isSecretValid(clientSecret: String): Boolean {
return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret) return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)

View file

@ -201,7 +201,7 @@ class ServerManager(
if (newSettings != null) { if (newSettings != null) {
LOGGER.info { "Server settings received: $newSettings" } LOGGER.info { "Server settings received: $newSettings" }
warmBasedOnSettings(newSettings) warnBasedOnSettings(newSettings)
if (!state.settings.logicalEqual(newSettings)) { if (!state.settings.logicalEqual(newSettings)) {
LOGGER.info { "Doing internal restart of HTTP server to refresh settings" } LOGGER.info { "Doing internal restart of HTTP server to refresh settings" }
@ -221,7 +221,7 @@ class ServerManager(
val remoteSettings = backendApi.loginToControl() val remoteSettings = backendApi.loginToControl()
?: throw RuntimeException("Failed to get a login response from server") ?: throw RuntimeException("Failed to get a login response from server")
LOGGER.info { "Server settings received: $remoteSettings" } LOGGER.info { "Server settings received: $remoteSettings" }
warmBasedOnSettings(remoteSettings) warnBasedOnSettings(remoteSettings)
val server = getServer( val server = getServer(
storage, storage,
@ -287,7 +287,7 @@ class ServerManager(
LOGGER.info { "Image server has shut down" } LOGGER.info { "Image server has shut down" }
} }
private fun warmBasedOnSettings(settings: RemoteSettings) { private fun warnBasedOnSettings(settings: RemoteSettings) {
if (settings.latestBuild > Constants.CLIENT_BUILD) { if (settings.latestBuild > Constants.CLIENT_BUILD) {
LOGGER.warn { LOGGER.warn {
"Outdated build detected! Latest: ${settings.latestBuild}, Current: ${Constants.CLIENT_BUILD}" "Outdated build detected! Latest: ${settings.latestBuild}, Current: ${Constants.CLIENT_BUILD}"

View file

@ -207,11 +207,7 @@ class ImageStorage(
return null return null
} }
return try { return WriterImpl(id, metadata)
WriterImpl(id, metadata)
} catch (e: FileAlreadyExistsException) {
null
}
} }
private fun deleteImage(id: String) { private fun deleteImage(id: String) {
@ -317,15 +313,28 @@ class ImageStorage(
} }
} catch (e: SQLIntegrityConstraintViolationException) { } catch (e: SQLIntegrityConstraintViolationException) {
// someone got to us before this (TOCTOU) // someone got to us before this (TOCTOU)
abort() // there are 2 situations here
return false // one is that the
// other write died in between writing the DB and
// moving the file
// the other is that we have raced and the other
// is about to write the file
// we handle this below
} }
try {
Files.move( Files.move(
tempPath, tempPath,
getPath(id), getPath(id),
StandardCopyOption.ATOMIC_MOVE StandardCopyOption.ATOMIC_MOVE
) )
} catch (e: FileAlreadyExistsException) {
// the file already exists
// so we must lost the race
// delete our local copy
abort()
return false
}
return true return true
} }
@ -333,12 +342,6 @@ class ImageStorage(
stream.flush() stream.flush()
stream.close() stream.close()
Files.deleteIfExists(tempPath) Files.deleteIfExists(tempPath)
// remove the database entry if it somehow exists
// this really shouldn't happen but just in case
database.delete(DbImage) {
DbImage.id eq id
}
} }
} }

View file

@ -24,6 +24,7 @@ import com.maxmind.geoip2.DatabaseReader
import com.maxmind.geoip2.exception.GeoIp2Exception import com.maxmind.geoip2.exception.GeoIp2Exception
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.logging.debug import mdnet.logging.debug
import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils import org.apache.commons.io.IOUtils
import org.http4k.core.Filter import org.http4k.core.Filter
@ -67,9 +68,9 @@ class GeoIpMetricsFilter(
} }
} catch (e: GeoIp2Exception) { } catch (e: GeoIp2Exception) {
// do not disclose ip here, for privacy of logs // do not disclose ip here, for privacy of logs
LOGGER.warn("Cannot resolve the country of the request's IP!") LOGGER.warn { "Cannot resolve the country of the request's IP!" }
} catch (e: UnknownHostException) { } catch (e: UnknownHostException) {
LOGGER.warn("Cannot resolve source IP of the request!") LOGGER.warn { "Cannot resolve source IP of the request!" }
} }
} }
} }

View file

@ -21,6 +21,9 @@ package mdnet.netty
import io.netty.bootstrap.ServerBootstrap import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.* import io.netty.channel.*
import io.netty.channel.epoll.Epoll
import io.netty.channel.epoll.EpollEventLoopGroup
import io.netty.channel.epoll.EpollServerSocketChannel
import io.netty.channel.nio.NioEventLoopGroup import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.SocketChannel import io.netty.channel.socket.SocketChannel
import io.netty.channel.socket.nio.NioServerSocketChannel import io.netty.channel.socket.nio.NioServerSocketChannel
@ -34,6 +37,9 @@ import io.netty.handler.timeout.WriteTimeoutException
import io.netty.handler.timeout.WriteTimeoutHandler import io.netty.handler.timeout.WriteTimeoutHandler
import io.netty.handler.traffic.GlobalTrafficShapingHandler import io.netty.handler.traffic.GlobalTrafficShapingHandler
import io.netty.handler.traffic.TrafficCounter import io.netty.handler.traffic.TrafficCounter
import io.netty.incubator.channel.uring.IOUring
import io.netty.incubator.channel.uring.IOUringEventLoopGroup
import io.netty.incubator.channel.uring.IOUringServerSocketChannel
import io.netty.util.concurrent.DefaultEventExecutorGroup import io.netty.util.concurrent.DefaultEventExecutorGroup
import mdnet.Constants import mdnet.Constants
import mdnet.data.Statistics import mdnet.data.Statistics
@ -57,17 +63,72 @@ import java.security.cert.X509Certificate
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
import javax.net.ssl.SSLException import javax.net.ssl.SSLException
interface NettyTransport {
val masterGroup: EventLoopGroup
val workerGroup: EventLoopGroup
val factory: ChannelFactory<ServerChannel>
fun shutdownGracefully() {
masterGroup.shutdownGracefully()
workerGroup.shutdownGracefully()
}
private class NioTransport : NettyTransport {
override val masterGroup = NioEventLoopGroup()
override val workerGroup = NioEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { NioServerSocketChannel() }
}
private class EpollTransport : NettyTransport {
override val masterGroup = EpollEventLoopGroup()
override val workerGroup = EpollEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { EpollServerSocketChannel() }
}
private class IOUringTransport : NettyTransport {
override val masterGroup = IOUringEventLoopGroup()
override val workerGroup = IOUringEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { IOUringServerSocketChannel() }
}
companion object {
private val LOGGER = LoggerFactory.getLogger(NettyTransport::class.java)
fun bestForPlatform(): NettyTransport {
if (IOUring.isAvailable()) {
LOGGER.info("Using IOUring transport")
return IOUringTransport()
} else {
LOGGER.info(IOUring.unavailabilityCause()) {
"IOUring transport not available"
}
}
if (Epoll.isAvailable()) {
LOGGER.info("Using Epoll transport")
return EpollTransport()
} else {
LOGGER.info(Epoll.unavailabilityCause()) {
"Epoll transport not available"
}
}
LOGGER.info("Using Nio transport")
return NioTransport()
}
}
}
class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig { class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer { override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
private val masterGroup = NioEventLoopGroup() private val transport = NettyTransport.bestForPlatform()
private val workerGroup = NioEventLoopGroup()
private val executor = DefaultEventExecutorGroup(serverSettings.threads) private val executor = DefaultEventExecutorGroup(serverSettings.threads)
private lateinit var closeFuture: ChannelFuture private lateinit var closeFuture: ChannelFuture
private lateinit var address: InetSocketAddress private lateinit var address: InetSocketAddress
private val burstLimiter = object : GlobalTrafficShapingHandler( private val burstLimiter = object : GlobalTrafficShapingHandler(
workerGroup, serverSettings.maxKilobitsPerSecond * 1000L / 8L, 0, 50 transport.workerGroup, serverSettings.maxKilobitsPerSecond * 1000L / 8L, 0, 50
) { ) {
override fun doAccounting(counter: TrafficCounter) { override fun doAccounting(counter: TrafficCounter) {
statistics.getAndUpdate { statistics.getAndUpdate {
@ -87,8 +148,8 @@ class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings
.build() .build()
val bootstrap = ServerBootstrap() val bootstrap = ServerBootstrap()
bootstrap.group(masterGroup, workerGroup) bootstrap.group(transport.masterGroup, transport.workerGroup)
.channelFactory(ChannelFactory<ServerChannel> { NioServerSocketChannel() }) .channelFactory(transport.factory)
.childHandler(object : ChannelInitializer<SocketChannel>() { .childHandler(object : ChannelInitializer<SocketChannel>() {
public override fun initChannel(ch: SocketChannel) { public override fun initChannel(ch: SocketChannel) {
ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc())) ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc()))
@ -133,8 +194,7 @@ class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings
override fun stop() = apply { override fun stop() = apply {
closeFuture.cancel(false) closeFuture.cancel(false)
workerGroup.shutdownGracefully() transport.shutdownGracefully()
masterGroup.shutdownGracefully()
executor.shutdownGracefully() executor.shutdownGracefully()
} }

View file

@ -13,6 +13,7 @@
</rollingPolicy>--> </rollingPolicy>-->
<encoder> <encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern> <pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
</encoder> </encoder>
</appender> </appender>
@ -23,6 +24,7 @@
</filter> </filter>
<encoder> <encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern> <pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
</encoder> </encoder>
</appender> </appender>

View file

@ -289,6 +289,11 @@ class TokenVerifierTest : FreeSpec() {
val response = handler(Request(Method.GET, "/a/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) val response = handler(Request(Method.GET, "/a/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
response.shouldNotHaveStatus(Status.OK) response.shouldNotHaveStatus(Status.OK)
} }
"invalid token should fail" {
val response = handler(Request(Method.GET, "/MTIzM2Vhd2Z3YWVmbG1pbzJuM29pNG4yaXAzNG1wMSwyWzMscHdxZWVlZWVlZXBscWFkcVt3ZGwxWzJsM3BbMWwycFsxZSxwMVssZmRbcGF3LGZwW2F3ZnBbLA==/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
response.shouldNotHaveStatus(Status.OK)
}
} }
} }
} }