Merge branch 'prom' into 'next-gen'

Prom+Grafana premade docker compose and config

See merge request mangadex-pub/mangadex_at_home!68

.gitignore

@@ -107,4 +107,8 @@ log/**
 images/**
 *.db
 
-settings.json
+*settings.json
+
+/cache
+docker/data
+data.mv.db

.gitlab-ci.yml

@@ -44,4 +44,4 @@ publish_docker:
     - mv build/libs/mangadex_at_home-${VERSION}-all.jar build/libs/mangadex_at_home.jar
     - docker build -t ${CI_REGISTRY_IMAGE}:${VERSION} -t ${CI_REGISTRY_IMAGE}:latest .
     - docker push ${CI_REGISTRY_IMAGE}:${VERSION}
-    - docker push ${CI_REGISTRY_IMAGE}:latest
+    - docker push ${CI_REGISTRY_IMAGE}:latest

Dockerfile

@@ -1,7 +1,13 @@
-FROM openjdk:15-alpine
+FROM adoptopenjdk:15
+
 WORKDIR /mangahome
-COPY /build/libs/mangadex_at_home.jar .
-RUN apk update
-VOLUME "/mangahome/cache"
+ADD /build/libs/mangadex_at_home.jar /mangahome/mangadex_at_home.jar
+
 EXPOSE 443 8080
-CMD java -Dfile-level=off -Dstdout-level=trace -jar mangadex_at_home.jar
+
+STOPSIGNAL 2
+
+CMD exec java \
+    -Dfile-level=off \
+    -Dstdout-level=info \
+    -jar mangadex_at_home.jar

README.md

@@ -11,3 +11,5 @@
 
 - Run `./gradlew build` in order to build the entire project
 - Find the generated jars in `build/libs`, where the `-all` jar is fat-jar with all dependencies
+
+### Run with [Docker](docker) (& optionally Prometheus+Grafana)

build.gradle

@@ -27,18 +27,21 @@ dependencies {
     compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
 
     implementation group: "commons-io", name: "commons-io", version: "2.7"
+    implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
     implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
 
+    implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
+    implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
 
     implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
     implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
     implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1"
     implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
     implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
     implementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version"
     runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
 
-
     implementation group: "com.h2database", name: "h2", version: "1.4.200"
     implementation "org.ktorm:ktorm-core:$ktorm_version"
     implementation "org.ktorm:ktorm-jackson:$ktorm_version"
@@ -48,6 +51,7 @@ dependencies {
 
     testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
     testImplementation "io.kotest:kotest-assertions-core:$kotest_version"
+    testImplementation "org.junit.jupiter:junit-jupiter:5.4.2"
     testImplementation "io.mockk:mockk:1.10.4"
 }
 
@@ -140,6 +144,6 @@ def listConfigurationDependencies(Configuration configuration) {
                 }
     } else {
         out << 'No dependencies found'
-    }   
+    }
     println(out)
 }

docker/README.md

@@ -0,0 +1,61 @@
+# Run with Docker
+
+⚠ This is a bit more involved of a setup than just running the jar ⚠
+
+## Prerequisites
+
+Docker Desktop for your operating system.
+
+Once installed, you can check that it works by opening a command prompt and running
+
+    docker run -it hello-world
+
+## Run as a standalone container
+
+Use either a specific image, preferrably the [latest image published](https://gitlab.com/mangadex-pub/mangadex_at_home/container_registry/1200259)
+
+> While it might work, using `registry.gitlab.com/mangadex-pub/mangadex_at_home:latest` is a bad idea as we do not guarantee forward-compatibility
+
+## Run with Prometheus and Grafana (i.e. dashboards)
+
+![](dashboard.png)
+
+### Quickstart
+
+1. Install `docker-compose`. Follow the steps [here](https://docs.docker.com/compose/install/)
+
+2. Copy the `docker` directory somewhere *on the drive you want to use as cache storage**
+
+   a. edit `docker-compose.yml` and replace `registry.gitlab.com/mangadex-pub/mangadex_at_home:<version>` with the appropriate version
+
+3. Copy your `settings.json` inside that directory (it should be next to `docker-compose.yml`)
+
+4. Run `docker-compose up -d` from within this directory
+
+5. That's it. You should now check the following:
+    - There are 3 containers in 'Up' state when running `docker ps` (`mangadex-at-home`, `prometheus` and `grafana`)
+    - The test image loads at [https://localhost/data/8172a46adc798f4f4ace6663322a383e/B18.png](https://localhost/data/8172a46adc798f4f4ace6663322a383e/B18.png)
+    - Prometheus loads at [http://localhost:9090](http://localhost:9090)
+    - Grafana loads at [http://localhost:3000/dashboards](http://localhost:3000/dashboards) and you can open the dashboard
+
+### Notes
+
+The pre-made configuration is hardcoded both public port 443 and this directory structure:
+
+    <directory where you run 'docker-compose up'>
+
+        Folders/files copied from the git repository
+        -> prometheus/... - pre-made config
+        -> grafana/... - pre-made config
+        -> docker-compose.yml
+
+        Your settings.json
+        -> settings.json
+
+        Created by the containers
+        -> data/
+            -> cache - the client's image cache
+            -> prometheus - prometheus database files
+            -> grafana - grafana files
+
+All of this is configurable to suit your needs but is not recommended unless you are familiar with Docker already.

docker/docker-compose.yml

@@ -0,0 +1,72 @@
+version: '3.8'
+
+services:
+
+  mangadex-at-home:
+    container_name: mangadex-at-home
+    image: "registry.gitlab.com/mangadex-pub/mangadex_at_home:<version>"
+    ports:
+      - 443:443
+    volumes:
+      - ./settings.json:/mangahome/settings.json:ro
+      - ./data/cache/:/mangahome/data/
+    environment:
+      JAVA_TOOL_OPTIONS: "-Xms1G -Xmx1G -XX:+UseG1GC -Xss512K"
+    command: [
+        "bash",
+        "-c",
+        "java \
+          -Dfile-level=off \
+          -Dstdout-level=info \
+          -jar mangadex_at_home.jar \
+          --cache /mangahome/data/images \
+          --database /mangahome/data/metadata"
+    ]
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "2"
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus
+    user: "root"
+    group_add:
+      - 0
+    ports:
+      - 9090:9090
+    links:
+      - mangadex-at-home
+    volumes:
+      - ./prometheus/:/etc/prometheus/:ro
+      - ./data/prometheus/:/prometheus/
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "2"
+
+  grafana:
+    container_name: grafana
+    image: grafana/grafana
+    user: "root"
+    group_add:
+      - 0
+    ports:
+      - 3000:3000
+    links:
+      - prometheus
+    volumes:
+      - ./grafana/:/etc/grafana/:ro
+      - ./data/grafana/:/var/lib/grafana/
+    environment:
+      GF_INSTALL_PLUGINS: "grafana-worldmap-panel"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "2"
+
+networks:
+  mangadex-at-home: { }

docker/grafana/grafana.ini

@@ -0,0 +1,8 @@
+[auth.anonymous]
+enabled = true
+
+# Organization name that should be used for unauthenticated users
+org_name = Main Org.
+
+# Role for unauthenticated users, other valid values are `Editor` and `Admin`
+org_role = Admin

docker/grafana/provisioning/dashboards/mdah-provider.yaml

@@ -0,0 +1,18 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'MangaDex@Home dashboards provider'
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: true
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: false
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/dashboards
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true

docker/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    version: 1
+    editable: false

docker/prometheus/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 20s
+  scrape_timeout: 10s
+
+scrape_configs:
+  - job_name: 'mangadex-at-home'
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+    metrics_path: /prometheus
+    static_configs:
+      - targets:
+          - "mangadex-at-home:443"

settings.sample.json

@@ -12,5 +12,10 @@
                                           // This rounds down to 15-second increments
     "max_kilobits_per_second": 0, // 0 disables max brust limiting
     "max_mebibytes_per_hour": 0   // 0 disables hourly bandwidth limiting
+  },
+
+  "metrics_settings": {
+    "enable_geoip": false, // set to true to get geoip metrics
+    "geoip_license_key": "unset" // if geoip metrics are enabled, a license is required, see https://dev.maxmind.com/geoip/geoip2/geolite2/
   }
 }

src/main/kotlin/mdnet/MangaDexClient.kt

@@ -30,7 +30,6 @@ import mdnet.cache.ImageStorage
 import mdnet.logging.info
 import mdnet.logging.warn
 import mdnet.settings.ClientSettings
-import org.http4k.server.Http4kServer
 import org.ktorm.database.Database
 import org.slf4j.LoggerFactory
 import java.io.File
@@ -106,7 +105,13 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo
             throw AssertionError()
         }
         LOGGER.info { "Server manager starting" }
-        imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, storage).also {
+        imageServer = ServerManager(
+            settings.serverSettings,
+            settings.devSettings,
+            settings.maxCacheSizeInMebibytes,
+            settings.metricsSettings,
+            storage
+        ).also {
             it.start()
         }
         LOGGER.info { "Server manager started" }
@@ -162,7 +167,8 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo
             storage.maxSize = (newSettings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong()
 
             val restartServer = newSettings.serverSettings != settings.serverSettings ||
-                newSettings.devSettings != settings.devSettings
+                newSettings.devSettings != settings.devSettings ||
+                newSettings.metricsSettings != settings.metricsSettings
 
             if (restartServer) {
                 stopImageServer()

src/main/kotlin/mdnet/ServerManager.kt

@@ -19,13 +19,17 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 /* ktlint-disable no-wildcard-imports */
 package mdnet
 
+import io.micrometer.prometheus.PrometheusConfig
+import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
 import mdnet.logging.error
 import mdnet.logging.info
 import mdnet.logging.warn
+import mdnet.metrics.DefaultMicrometerMetrics
 import mdnet.server.getServer
 import mdnet.settings.DevSettings
+import mdnet.settings.MetricsSettings
 import mdnet.settings.RemoteSettings
 import mdnet.settings.ServerSettings
 import org.http4k.server.Http4kServer
@@ -40,18 +44,34 @@ import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 
 sealed class State
+
 // server is not running
 data class Uninitialized(val serverSettings: ServerSettings, val devSettings: DevSettings) : State()
+
 // server has shut down
 object Shutdown : State()
+
 // server is in the process of stopping
-data class GracefulStop(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized(lastRunning.serverSettings, lastRunning.devSettings), val action: () -> Unit = {}) : State()
+data class GracefulStop(
+    val lastRunning: Running,
+    val counts: Int = 0,
+    val nextState: State = Uninitialized(lastRunning.serverSettings, lastRunning.devSettings),
+    val action: () -> Unit = {}
+) : State()
+
 // server is currently running
 data class Running(val server: Http4kServer, val settings: RemoteSettings, val serverSettings: ServerSettings, val devSettings: DevSettings) : State()
 
-class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val storage: ImageStorage) {
+class ServerManager(
+    serverSettings: ServerSettings,
+    devSettings: DevSettings,
+    maxCacheSizeInMebibytes: Long,
+    private val metricsSettings: MetricsSettings,
+    private val storage: ImageStorage
+) {
     // this must remain single-threaded because of how the state mechanism works
     private val executor = Executors.newSingleThreadScheduledExecutor()
+    private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
 
     // state that must only be accessed from the thread on the executor
     private var state: State
@@ -85,6 +105,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
 
     fun start() {
         LOGGER.info { "Image server starting" }
+        DefaultMicrometerMetrics(registry, storage.cacheDirectory)
         loginAndStartServer()
         statsMap[Instant.now()] = statistics.get()
 
@@ -238,7 +259,15 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
         LOGGER.info { "Server settings received: $remoteSettings" }
         warmBasedOnSettings(remoteSettings)
 
-        val server = getServer(storage, remoteSettings, state.serverSettings, statistics, isHandled).start()
+        val server = getServer(
+            storage,
+            remoteSettings,
+            state.serverSettings,
+            statistics,
+            isHandled,
+            metricsSettings,
+            registry
+        ).start()
 
         this.state = Running(server, remoteSettings, state.serverSettings, state.devSettings)
         LOGGER.info { "Internal HTTP server was successfully started" }

src/main/kotlin/mdnet/metrics/DefaultMicrometerMetrics.kt

@@ -0,0 +1,54 @@
+/*
+Mangadex@Home
+Copyright (c) 2020, MangaDex Network
+This file is part of MangaDex@Home.
+
+MangaDex@Home is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+MangaDex@Home is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/* ktlint-disable no-wildcard-imports */
+package mdnet.metrics
+
+import io.micrometer.core.instrument.Tag
+import io.micrometer.core.instrument.binder.jvm.DiskSpaceMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
+import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics
+import io.micrometer.core.instrument.binder.logging.LogbackMetrics
+import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics
+import io.micrometer.core.instrument.binder.system.ProcessorMetrics
+import io.micrometer.core.instrument.binder.system.UptimeMetrics
+import io.micrometer.prometheus.PrometheusMeterRegistry
+import mdnet.BuildInfo
+import java.nio.file.Path
+
+class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory: Path) {
+
+    init {
+        UptimeMetrics(
+            mutableListOf(
+                Tag.of("version", BuildInfo.VERSION)
+            )
+        ).bindTo(registry)
+
+        JvmMemoryMetrics().bindTo(registry)
+        JvmGcMetrics().bindTo(registry)
+        ProcessorMetrics().bindTo(registry)
+        JvmThreadMetrics().bindTo(registry)
+        JvmHeapPressureMetrics().bindTo(registry)
+        FileDescriptorMetrics().bindTo(registry)
+        LogbackMetrics().bindTo(registry)
+        DiskSpaceMetrics(cacheDirectory.toFile()).bindTo(registry)
+    }
+}

src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt

@@ -0,0 +1,142 @@
+/*
+Mangadex@Home
+Copyright (c) 2020, MangaDex Network
+This file is part of MangaDex@Home.
+
+MangaDex@Home is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+MangaDex@Home is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/* ktlint-disable no-wildcard-imports */
+package mdnet.metrics
+
+import com.maxmind.db.CHMCache
+import com.maxmind.geoip2.DatabaseReader
+import com.maxmind.geoip2.exception.GeoIp2Exception
+import io.micrometer.prometheus.PrometheusMeterRegistry
+import mdnet.logging.debug
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.io.IOUtils
+import org.http4k.core.Filter
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.Status
+import org.http4k.filter.gunzippedStream
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.net.InetAddress
+import java.nio.file.Files
+
+class GeoIpMetricsFilter(
+    private val databaseReader: DatabaseReader?,
+    private val registry: PrometheusMeterRegistry
+) : Filter {
+
+    companion object {
+        private val LOGGER: Logger = LoggerFactory.getLogger(GeoIpMetricsFilter::class.java)
+    }
+
+    override fun invoke(next: HttpHandler): HttpHandler {
+        return {
+            if (databaseReader != null && (it.uri.path != "/prometheus")) {
+                inspectAndRecordSourceCountry(it)
+            }
+
+            next(it)
+        }
+    }
+
+    private fun inspectAndRecordSourceCountry(request: Request) {
+        val sourceIp =
+            request.headerValues("Forwarded").firstOrNull() // try Forwarded (rare but standard)
+                ?: request.headerValues("X-Forwarded-For").firstOrNull() // X-Forwarded-For (common but technically wrong)
+                ?: request.source?.address // source (in case of no proxying, or with proxy-protocol)
+
+        sourceIp.apply {
+            try {
+                val inetAddress = InetAddress.getByName(sourceIp)
+                if (!inetAddress.isLoopbackAddress && !inetAddress.isAnyLocalAddress) {
+                    val country = databaseReader!!.country(inetAddress)
+                    recordCountry(country.country.isoCode)
+                }
+            } catch (ge: GeoIp2Exception) {
+                // do not disclose ip here, for privacy of logs
+                LOGGER.warn("Cannot resolve the country of the request's ip!")
+            } catch (e: Exception) {
+                LOGGER.warn("Cannot resolve source ip of the request!")
+            }
+        }
+    }
+
+    private fun recordCountry(code: String) {
+        registry.counter(
+            "requests_country_counts",
+            "country", code
+        ).increment()
+    }
+}
+
+class GeoIpMetricsFilterBuilder(
+    private val enableGeoIp: Boolean,
+    private val license: String,
+    private val registry: PrometheusMeterRegistry,
+    private val client: HttpHandler
+) {
+
+    private val LOGGER: Logger = LoggerFactory.getLogger(GeoIpMetricsFilterBuilder::class.java)
+
+    companion object {
+        const val GEOIP2_COUNTRY_URI_FORMAT: String =
+            "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=%s&suffix=tar.gz"
+    }
+
+    fun build(): GeoIpMetricsFilter {
+        return if (enableGeoIp) {
+            LOGGER.info("GeoIp is enabled. Initialising...")
+            val databaseReader = initDatabase()
+            LOGGER.info("GeoIp initialised")
+            GeoIpMetricsFilter(databaseReader, registry)
+        } else {
+            GeoIpMetricsFilter(null, registry)
+        }
+    }
+
+    private fun initDatabase(): DatabaseReader {
+        val databaseFileDir = Files.createTempDirectory("mangadex-geoip")
+        val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
+
+        val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
+        val response = client(Request(Method.GET, geoIpDatabaseUri))
+        if (response.status != Status.OK) {
+            throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")
+        }
+
+        response.use {
+            val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream)
+            var entry = archiveStream.nextTarEntry
+            while (!entry.name.endsWith(".mmdb")) {
+                LOGGER.debug { "Skip non-database file: ${entry.name}" }
+                entry = archiveStream.nextTarEntry
+            }
+
+            // reads only the current entry to its end
+            val dbBytes = IOUtils.toByteArray(archiveStream)
+            Files.write(databaseFile, dbBytes)
+        }
+
+        return DatabaseReader
+            .Builder(databaseFile.toFile())
+            .withCache(CHMCache())
+            .build()
+    }
+}

src/main/kotlin/mdnet/metrics/PostTransactionLabeler.kt

@@ -0,0 +1,37 @@
+/*
+Mangadex@Home
+Copyright (c) 2020, MangaDex Network
+This file is part of MangaDex@Home.
+
+MangaDex@Home is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+MangaDex@Home is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/* ktlint-disable no-wildcard-imports */
+package mdnet.metrics
+
+import org.http4k.core.HttpTransaction
+import org.http4k.filter.HttpTransactionLabeler
+
+class PostTransactionLabeler : HttpTransactionLabeler {
+
+    override fun invoke(transaction: HttpTransaction): HttpTransaction {
+        return transaction.copy(
+            labels = mapOf(
+                "method" to transaction.request.method.toString(),
+                "status" to transaction.response.status.code.toString(),
+                "path" to transaction.routingGroup,
+                "cache" to (transaction.response.header("X-Cache") ?: "MISS").toUpperCase()
+            )
+        )
+    }
+}

src/main/kotlin/mdnet/server/ImageServer.kt

@@ -25,6 +25,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
+import io.micrometer.core.instrument.FunctionCounter
+import io.micrometer.core.instrument.Timer
+import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.Constants
 import mdnet.cache.CachingInputStream
 import mdnet.cache.Image
@@ -35,8 +38,11 @@ import mdnet.data.Token
 import mdnet.logging.info
 import mdnet.logging.trace
 import mdnet.logging.warn
+import mdnet.metrics.GeoIpMetricsFilterBuilder
+import mdnet.metrics.PostTransactionLabeler
 import mdnet.netty.Netty
 import mdnet.security.TweetNaclFast
+import mdnet.settings.MetricsSettings
 import mdnet.settings.RemoteSettings
 import mdnet.settings.ServerSettings
 import org.apache.hc.client5.http.config.RequestConfig
@@ -48,6 +54,8 @@ import org.http4k.client.ApacheClient
 import org.http4k.core.*
 import org.http4k.filter.CachingFilters
 import org.http4k.filter.ClientFilters
+import org.http4k.filter.MicrometerMetrics
+import org.http4k.filter.ServerFilters
 import org.http4k.lens.LensFailure
 import org.http4k.lens.Path
 import org.http4k.routing.bind
@@ -60,7 +68,7 @@ import java.io.BufferedOutputStream
 import java.io.InputStream
 import java.time.Clock
 import java.time.OffsetDateTime
-import java.util.Base64
+import java.util.*
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
@@ -73,9 +81,14 @@ private val JACKSON: ObjectMapper = jacksonObjectMapper()
 class ImageServer(
     private val storage: ImageStorage,
     private val statistics: AtomicReference<Statistics>,
-    private val client: HttpHandler
+    private val client: HttpHandler,
+    registry: PrometheusMeterRegistry
 ) {
     private val executor = Executors.newCachedThreadPool()
+    private val cacheLookupTimer = Timer
+        .builder("cache_lookup")
+        .publishPercentiles(0.5, 0.75, 0.9, 0.99)
+        .register(registry)
 
     // This is part of the ImageServer , and it expects `chapterHash` and `fileName` path segments.
     fun handler(dataSaver: Boolean): HttpHandler = baseHandler().then { request ->
@@ -100,7 +113,7 @@ class ImageServer(
             printHexString(it)
         }
 
-        val image = storage.loadImage(imageId)
+        val image: Image? = cacheLookupTimer.recordCallable { storage.loadImage(imageId) }
 
         if (image != null) {
             request.handleCacheHit(sanitizedUri, image)
@@ -238,7 +251,15 @@ class ImageServer(
 
 private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/")
 
-fun getServer(storage: ImageStorage, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
+fun getServer(
+    storage: ImageStorage,
+    remoteSettings: RemoteSettings,
+    serverSettings: ServerSettings,
+    statistics: AtomicReference<Statistics>,
+    isHandled: AtomicBoolean,
+    metricsSettings: MetricsSettings,
+    registry: PrometheusMeterRegistry,
+): Http4kServer {
     val apache = ApacheClient(
         responseBodyMode = BodyMode.Stream,
         client = HttpClients.custom()
@@ -259,15 +280,26 @@ fun getServer(storage: ImageStorage, remoteSettings: RemoteSettings, serverSetti
             )
             .build()
     )
-    val client = ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
-        .then(apache)
+
+    val client =
+        ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
+            .then(ClientFilters.MicrometerMetrics.RequestCounter(registry))
+            .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
+            .then(apache)
 
     val imageServer = ImageServer(
         storage = storage,
         statistics = statistics,
-        client = client
+        client = client,
+        registry = registry
     )
 
+    FunctionCounter.builder(
+        "client_sent_bytes",
+        statistics,
+        { it.get().bytesSent.toDouble() }
+    ).register(registry)
+
     val verifier = tokenVerifier(
         tokenKey = remoteSettings.tokenKey,
         shouldVerify = { chapter, _ ->
@@ -301,6 +333,13 @@ fun getServer(storage: ImageStorage, remoteSettings: RemoteSettings, serverSetti
                         dataSaver = true,
                     )
                 ),
+                "/prometheus" bind Method.GET to {
+                    Response(Status.OK).body(registry.scrape())
+                }
+            ).withFilter(
+                ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
+            ).withFilter(
+                GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build()
             )
         )
         .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))
@@ -320,7 +359,7 @@ fun timeRequest(): Filter {
         { request: Request ->
             val cleanedUri = request.uri.path.replaceBefore("/data", "/{token}")
 
-            LOGGER.info { "Request for $cleanedUri received from ${request.source?.address}" }
+            LOGGER.info { "Request for $cleanedUri received" }
 
             val start = System.currentTimeMillis()
             val response = next(request)

src/main/kotlin/mdnet/settings/ClientSettings.kt

@@ -27,6 +27,7 @@ data class ClientSettings(
     val maxCacheSizeInMebibytes: Long = 20480,
     val devSettings: DevSettings = DevSettings(),
     val serverSettings: ServerSettings = ServerSettings(),
+    val metricsSettings: MetricsSettings = MetricsSettings(),
 )
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
@@ -45,3 +46,9 @@ data class ServerSettings(
 data class DevSettings(
     val devUrl: String? = null
 )
+
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
+data class MetricsSettings(
+    val enableGeoip: Boolean = false,
+    @field:Secret val geoipLicenseKey: String = "none"
+)

src/test/kotlin/mdnet/metrics/GeoIpMetricsFilterTest.kt

@@ -0,0 +1,108 @@
+package mdnet.metrics
+
+import com.maxmind.geoip2.DatabaseReader
+import com.maxmind.geoip2.model.CountryResponse
+import com.maxmind.geoip2.record.Country
+import io.micrometer.core.instrument.Counter
+import io.micrometer.prometheus.PrometheusMeterRegistry
+import io.mockk.confirmVerified
+import io.mockk.every
+import io.mockk.impl.annotations.MockK
+import io.mockk.junit5.MockKExtension
+import io.mockk.mockk
+import io.mockk.verify
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.RequestSource
+import org.http4k.core.Response
+import org.http4k.core.Status
+import org.http4k.kotest.shouldHaveStatus
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import java.net.InetAddress
+
+@ExtendWith(MockKExtension::class)
+internal class GeoIpMetricsFilterTest {
+
+    @MockK
+    private lateinit var registry: PrometheusMeterRegistry
+
+    @MockK
+    private lateinit var databaseReader: DatabaseReader
+
+    private lateinit var geoIpMetricsFilter: GeoIpMetricsFilter
+
+    @BeforeEach
+    fun inject() {
+        geoIpMetricsFilter = GeoIpMetricsFilter(databaseReader, registry)
+    }
+
+    @Test
+    fun `Invalid source doesn't fail the image serving`() {
+        val address = "not a resolvable inetaddress"
+
+        val request: Request = Request(Method.GET, "whatever")
+            .source(RequestSource(address = address))
+
+        val response = filterRequest(request)
+
+        response shouldHaveStatus Status.OK
+    }
+
+    @Test
+    fun `Invalid header doesn't fail the image serving`() {
+        val address = "not a resolvable inetaddress"
+
+        val request: Request = Request(Method.GET, "whatever")
+            .header("X-Forwarded-For", address)
+
+        val response = filterRequest(request)
+
+        response shouldHaveStatus Status.OK
+    }
+
+    @Test
+    fun `Valid header and country resolved`() {
+        val address = "195.154.69.12"
+        val countryCode = "COUNTRY_CODE"
+
+        val countryResponse = mockk<CountryResponse>()
+        val country = mockk<Country>()
+        val counter = mockk<Counter>(relaxUnitFun = true)
+
+        every { country.isoCode } returns countryCode
+        every { countryResponse.country } returns country
+        every { databaseReader.country(InetAddress.getByName(address)) } returns countryResponse
+        every {
+            registry.counter(
+                "requests_country_counts",
+                "country", countryCode
+            )
+        } returns counter
+
+        val request: Request = Request(Method.GET, "whatever")
+            .header("X-Forwarded-For", address)
+
+        val response = filterRequest(request)
+
+        response shouldHaveStatus Status.OK
+
+        verify {
+            registry.counter(
+                "requests_country_counts",
+                "country", countryCode
+            )
+        }
+        confirmVerified(registry)
+
+        verify {
+            counter.increment()
+        }
+        confirmVerified(counter)
+    }
+
+    private fun filterRequest(request: Request): Response {
+        return geoIpMetricsFilter.invoke(next = { Response(Status.OK) })(request)
+    }
+}

src/test/kotlin/mdnet/server/ImageServerTest.kt

@@ -25,6 +25,8 @@ import io.kotest.core.spec.style.FreeSpec
 import io.kotest.engine.spec.tempdir
 import io.kotest.engine.spec.tempfile
 import io.kotest.matchers.shouldBe
+import io.micrometer.prometheus.PrometheusConfig
+import io.micrometer.prometheus.PrometheusMeterRegistry
 import io.mockk.confirmVerified
 import io.mockk.every
 import io.mockk.mockk
@@ -34,7 +36,12 @@ import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
 import mdnet.security.TweetNaclFast
 import org.apache.commons.io.IOUtils
-import org.http4k.core.*
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.Response
+import org.http4k.core.Status
+import org.http4k.core.then
 import org.http4k.kotest.shouldHaveHeader
 import org.http4k.kotest.shouldHaveStatus
 import org.http4k.routing.bind
@@ -47,6 +54,7 @@ class ImageServerTest : FreeSpec() {
     override fun isolationMode() = IsolationMode.InstancePerTest
 
     init {
+        val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
         val mockData = byteArrayOf(72, 66, 67, 66, 65, 66, 73, 69, 65, 67)
 
         "correct upstream responses" - {
@@ -69,7 +77,8 @@ class ImageServerTest : FreeSpec() {
                 val server = ImageServer(
                     storage,
                     AtomicReference(Statistics()),
-                    client
+                    client,
+                    registry
                 )
 
                 val handler = routes(
@@ -117,7 +126,8 @@ class ImageServerTest : FreeSpec() {
                 val server = ImageServer(
                     storage,
                     AtomicReference(Statistics()),
-                    client
+                    client,
+                    registry
                 )
 
                 val handler = routes(
@@ -165,7 +175,8 @@ class ImageServerTest : FreeSpec() {
             val server = ImageServer(
                 storage,
                 AtomicReference(Statistics()),
-                client
+                client,
+                registry
             )
 
             val handler = routes(
@@ -261,7 +272,6 @@ class TokenVerifierTest : FreeSpec() {
         }
 
         "token" - {
-
         }
     }
-}
+}