gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Fix downloading of GeoIp2 database

Browse source
This commit is contained in:
Tristan Deloche 2021-01-22 01:12:41 +00:00
parent e8f17578de
commit d2949c6be2
No known key found for this signature in database
GPG key ID: B6BDB16331573E59
3 changed files with 32 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
build.gradle
View file

@ -27,6 +27,7 @@ dependencies {
compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
implementation group: "commons-io", name: "commons-io", version: "2.7"
implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
compile 'io.micrometer:micrometer-registry-prometheus:1.6.2'

67
src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt
View file

@ -5,15 +5,13 @@ import com.maxmind.geoip2.DatabaseReader
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.logging.info
import mdnet.logging.trace
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Method
import org.http4k.core.Request
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.http4k.core.*
import org.http4k.filter.gunzippedStream
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.net.InetAddress
import java.nio.file.Files
import java.nio.file.Path
class GeoIpMetricsFilter(
private val enableGeoIp: Boolean,
@ -22,13 +20,13 @@ class GeoIpMetricsFilter(
private val client: HttpHandler
) : Filter {
private val logger: Logger = LoggerFactory.getLogger(GeoIpMetricsFilter::class.java)
private val LOGGER: Logger = LoggerFactory.getLogger(GeoIpMetricsFilter::class.java)
private lateinit var dbr: DatabaseReader
init {
if (enableGeoIp) {
logger.info("Initializing GeoIp support!")
LOGGER.info { "Initializing GeoIp support!" }
dbr = initDatabase()
}
}
@ -39,17 +37,21 @@ class GeoIpMetricsFilter(
}
return {
val source = it.header("X-Forwarded-For") ?: it.source?.address
source.apply {
val sourceIp =
it.headerValues("Forwarded")[0] // try Forwarded (rare but standard)
?: it.headerValues("X-Forwarded-For")[0] // X-Forwarded-For (common but technically wrong)
?: it.source?.address // source (in case of no proxying, or with proxy-protocol)
sourceIp.apply {
try {
val inetAddress = InetAddress.getByName(source)
val inetAddress = InetAddress.getByName(sourceIp)
if (!inetAddress.isLoopbackAddress && !inetAddress.isAnyLocalAddress) {
val country = dbr.country(inetAddress)
recordCountry(country.country.isoCode)
}
} catch (e: Exception) {
// do not disclose ip here, for privacy of logs
logger.warn("Cannot resolve country from ip!")
LOGGER.warn("Cannot resolve country from ip!")
}
}
@ -58,7 +60,7 @@ class GeoIpMetricsFilter(
}
private fun recordCountry(code: String) {
logger.trace { "Registered hit for country code: $code" }
LOGGER.trace { "Registered hit for country code: $code" }
registry.counter(
"requests_country_counts",
"country", code
@ -67,17 +69,26 @@ class GeoIpMetricsFilter(
private fun initDatabase(): DatabaseReader {
val databaseFileDir = Files.createTempDirectory("mangadex-geoip")
val downloadLocation = Files.createTempFile(databaseFileDir, "download", ".tar.gz")
val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
val geoIpDatabaseUri = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=%s&suffix=tar.gz".format(license)
val request = Request(Method.GET, geoIpDatabaseUri)
client(request).use {
Files.write(downloadLocation, it.body.payload.array())
val response = client(Request(Method.GET, geoIpDatabaseUri))
if (response.status != Status.OK) {
throw IllegalStateException("Couldn't download GeoIP 2 databse (status code from upstream: ${response.status})")
}
logger.info { "Downloaded GeoIp database at $downloadLocation" }
val databaseFile: Path = untarDatabase(databaseFileDir, downloadLocation)
response.use {
val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream)
var entry = archiveStream.nextTarEntry
while (!entry.name.endsWith(".mmdb")) {
LOGGER.trace { "Skip non-database file: ${entry.name}" }
entry = archiveStream.nextTarEntry
}
// reads only the current entry to its end
val dbBytes = archiveStream.readAllBytes()
Files.write(databaseFile, dbBytes)
}
return DatabaseReader
.Builder(databaseFile.toFile())
@ -85,22 +96,4 @@ class GeoIpMetricsFilter(
.build()
}
private fun untarDatabase(cwd: Path, tarFile: Path): Path {
val untarProcess = ProcessBuilder()
.directory(cwd.toFile())
.command("/usr/bin/env", "tar", "xfv", tarFile.toAbsolutePath().toString())
.inheritIO()
.start()
val retCode = untarProcess.waitFor()
logger.info { "Tar return code: $retCode" }
return Files
.walk(cwd)
.filter { path: Path -> path.fileName.toString().endsWith(".mmdb") }
.findAny()
.orElseThrow()
.toAbsolutePath()
}
}

2
src/main/kotlin/mdnet/server/ImageServer.kt
View file

@ -326,7 +326,7 @@ fun getServer(
).withFilter(
ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
).withFilter(
GeoIpMetricsFilter(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, client)
GeoIpMetricsFilter(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache)
)
)
.asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))