gitlab-com-mangadex-pub
/
mangadex_at_home
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git
1
0
Fork 1
Code Releases Activity

Fix a bunch of various nits

This commit is contained in:
carbotaniuman 2021-01-24 12:05:05 -06:00
parent dc53eea42f
commit 5ee6dd23cd
11 changed files with 122 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -107,7 +107,7 @@ log/**
images/**
*.db
*settings.yaml
settings.yaml
/cache
docker/data

18
CHANGELOG.md
View File

@ -14,11 +14,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
### Fixed
- [2021-01-23] Fix sample docker-compose.yml [@_tde9]
### Security
## [2.0.0-rc1] - 2020-01-23
## [2.0.0-rc2] - 2021-01-24
### Added
- [2021-01-24] Epoll and IOUring HTTP transports [@carbotaniuman].
### Changed
- [2021-01-23] Log files are now UTF-8 [@_tde9]
### Fixed
- [2021-01-23] Fix sample docker-compose.yml [@_tde9]
- [2021-01-24] Fix another race condition [@carbotaniuman]
- [2021-01-23] Fix validating small (<1024MiB) caches [@_tde9]
## [2.0.0-rc1] - 2021-01-23
This release contains many breaking changes! Of note are the changes to the cache folders, database location, and settings format.
### Added
- [2021-01-23] Added `external_max_kilobits_per_second` config option [@carbotaniuman].
@ -280,7 +291,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc1...HEAD
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc2...HEAD
[2.0.0-rc2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc1...2.0.0-rc2
[2.0.0-rc1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.4...2.0.0-rc1
[1.2.4]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.3...1.2.4
[1.2.3]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/1.2.2...1.2.3

2
build.gradle
View File

@ -39,6 +39,8 @@ dependencies {
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"

2
src/main/kotlin/mdnet/Constants.kt
View File

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 21
const val CLIENT_BUILD = 22
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

6
src/main/kotlin/mdnet/MangaDexClient.kt
View File

@ -188,9 +188,9 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo
}
private fun validateSettings(settings: ClientSettings) {
// if (settings.maxCacheSizeInMebibytes < 1024) {
// throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
// }
if (settings.maxCacheSizeInMebibytes < 1024) {
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
}
fun isSecretValid(clientSecret: String): Boolean {
return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)

6
src/main/kotlin/mdnet/ServerManager.kt
View File

@ -201,7 +201,7 @@ class ServerManager(
if (newSettings != null) {
LOGGER.info { "Server settings received: $newSettings" }
warmBasedOnSettings(newSettings)
warnBasedOnSettings(newSettings)
if (!state.settings.logicalEqual(newSettings)) {
LOGGER.info { "Doing internal restart of HTTP server to refresh settings" }
@ -221,7 +221,7 @@ class ServerManager(
val remoteSettings = backendApi.loginToControl()
?: throw RuntimeException("Failed to get a login response from server")
LOGGER.info { "Server settings received: $remoteSettings" }
warmBasedOnSettings(remoteSettings)
warnBasedOnSettings(remoteSettings)
val server = getServer(
storage,
@ -287,7 +287,7 @@ class ServerManager(
LOGGER.info { "Image server has shut down" }
}
private fun warmBasedOnSettings(settings: RemoteSettings) {
private fun warnBasedOnSettings(settings: RemoteSettings) {
if (settings.latestBuild > Constants.CLIENT_BUILD) {
LOGGER.warn {
"Outdated build detected! Latest: ${settings.latestBuild}, Current: ${Constants.CLIENT_BUILD}"

37
src/main/kotlin/mdnet/cache/ImageStorage.kt vendored
View File

@ -207,11 +207,7 @@ class ImageStorage(
return null
}
return try {
WriterImpl(id, metadata)
} catch (e: FileAlreadyExistsException) {
null
}
return WriterImpl(id, metadata)
}
private fun deleteImage(id: String) {
@ -317,15 +313,28 @@ class ImageStorage(
}
} catch (e: SQLIntegrityConstraintViolationException) {
// someone got to us before this (TOCTOU)
// there are 2 situations here
// one is that the
// other write died in between writing the DB and
// moving the file
// the other is that we have raced and the other
// is about to write the file
// we handle this below
}
try {
Files.move(
tempPath,
getPath(id),
StandardCopyOption.ATOMIC_MOVE
)
} catch (e: FileAlreadyExistsException) {
// the file already exists
// so we must lost the race
// delete our local copy
abort()
return false
}
Files.move(
tempPath,
getPath(id),
StandardCopyOption.ATOMIC_MOVE
)
return true
}
@ -333,12 +342,6 @@ class ImageStorage(
stream.flush()
stream.close()
Files.deleteIfExists(tempPath)
// remove the database entry if it somehow exists
// this really shouldn't happen but just in case
database.delete(DbImage) {
DbImage.id eq id
}
}
}

5
src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt
View File

@ -24,6 +24,7 @@ import com.maxmind.geoip2.DatabaseReader
import com.maxmind.geoip2.exception.GeoIp2Exception
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.logging.debug
import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils
import org.http4k.core.Filter
@ -67,9 +68,9 @@ class GeoIpMetricsFilter(
}
} catch (e: GeoIp2Exception) {
// do not disclose ip here, for privacy of logs
LOGGER.warn("Cannot resolve the country of the request's IP!")
LOGGER.warn { "Cannot resolve the country of the request's IP!" }
} catch (e: UnknownHostException) {
LOGGER.warn("Cannot resolve source IP of the request!")
LOGGER.warn { "Cannot resolve source IP of the request!" }
}
}
}

74
src/main/kotlin/mdnet/netty/ApplicationNetty.kt
View File

@ -21,6 +21,9 @@ package mdnet.netty
import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.*
import io.netty.channel.epoll.Epoll
import io.netty.channel.epoll.EpollEventLoopGroup
import io.netty.channel.epoll.EpollServerSocketChannel
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.SocketChannel
import io.netty.channel.socket.nio.NioServerSocketChannel
@ -34,6 +37,9 @@ import io.netty.handler.timeout.WriteTimeoutException
import io.netty.handler.timeout.WriteTimeoutHandler
import io.netty.handler.traffic.GlobalTrafficShapingHandler
import io.netty.handler.traffic.TrafficCounter
import io.netty.incubator.channel.uring.IOUring
import io.netty.incubator.channel.uring.IOUringEventLoopGroup
import io.netty.incubator.channel.uring.IOUringServerSocketChannel
import io.netty.util.concurrent.DefaultEventExecutorGroup
import mdnet.Constants
import mdnet.data.Statistics
@ -57,17 +63,72 @@ import java.security.cert.X509Certificate
import java.util.concurrent.atomic.AtomicReference
import javax.net.ssl.SSLException
interface NettyTransport {
val masterGroup: EventLoopGroup
val workerGroup: EventLoopGroup
val factory: ChannelFactory<ServerChannel>
fun shutdownGracefully() {
masterGroup.shutdownGracefully()
workerGroup.shutdownGracefully()
}
private class NioTransport : NettyTransport {
override val masterGroup = NioEventLoopGroup()
override val workerGroup = NioEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { NioServerSocketChannel() }
}
private class EpollTransport : NettyTransport {
override val masterGroup = EpollEventLoopGroup()
override val workerGroup = EpollEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { EpollServerSocketChannel() }
}
private class IOUringTransport : NettyTransport {
override val masterGroup = IOUringEventLoopGroup()
override val workerGroup = IOUringEventLoopGroup()
override val factory = ChannelFactory<ServerChannel> { IOUringServerSocketChannel() }
}
companion object {
private val LOGGER = LoggerFactory.getLogger(NettyTransport::class.java)
fun bestForPlatform(): NettyTransport {
if (IOUring.isAvailable()) {
LOGGER.info("Using IOUring transport")
return IOUringTransport()
} else {
LOGGER.info(IOUring.unavailabilityCause()) {
"IOUring transport not available"
}
}
if (Epoll.isAvailable()) {
LOGGER.info("Using Epoll transport")
return EpollTransport()
} else {
LOGGER.info(Epoll.unavailabilityCause()) {
"Epoll transport not available"
}
}
LOGGER.info("Using Nio transport")
return NioTransport()
}
}
}
class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
private val masterGroup = NioEventLoopGroup()
private val workerGroup = NioEventLoopGroup()
private val transport = NettyTransport.bestForPlatform()
private val executor = DefaultEventExecutorGroup(serverSettings.threads)
private lateinit var closeFuture: ChannelFuture
private lateinit var address: InetSocketAddress
private val burstLimiter = object : GlobalTrafficShapingHandler(
workerGroup, serverSettings.maxKilobitsPerSecond * 1000L / 8L, 0, 50
transport.workerGroup, serverSettings.maxKilobitsPerSecond * 1000L / 8L, 0, 50
) {
override fun doAccounting(counter: TrafficCounter) {
statistics.getAndUpdate {
@ -87,8 +148,8 @@ class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings
.build()
val bootstrap = ServerBootstrap()
bootstrap.group(masterGroup, workerGroup)
.channelFactory(ChannelFactory<ServerChannel> { NioServerSocketChannel() })
bootstrap.group(transport.masterGroup, transport.workerGroup)
.channelFactory(transport.factory)
.childHandler(object : ChannelInitializer<SocketChannel>() {
public override fun initChannel(ch: SocketChannel) {
ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc()))
@ -133,8 +194,7 @@ class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings
override fun stop() = apply {
closeFuture.cancel(false)
workerGroup.shutdownGracefully()
masterGroup.shutdownGracefully()
transport.shutdownGracefully()
executor.shutdownGracefully()
}

2
src/main/resources/logback.xml
View File

@ -13,6 +13,7 @@
</rollingPolicy>-->
<encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
@ -23,6 +24,7 @@
</filter>
<encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>

5
src/test/kotlin/mdnet/server/ImageServerTest.kt
View File

@ -289,6 +289,11 @@ class TokenVerifierTest : FreeSpec() {
val response = handler(Request(Method.GET, "/a/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
response.shouldNotHaveStatus(Status.OK)
}
"invalid token should fail" {
val response = handler(Request(Method.GET, "/MTIzM2Vhd2Z3YWVmbG1pbzJuM29pNG4yaXAzNG1wMSwyWzMscHdxZWVlZWVlZXBscWFkcVt3ZGwxWzJsM3BbMWwycFsxZSxwMVssZmRbcGF3LGZwW2F3ZnBbLA==/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
response.shouldNotHaveStatus(Status.OK)
}
}
}
}