gitlab-com-mangadex-pub
/
mangadex_at_home
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git
1
0
Fork 1
Code Releases Activity

Update metrics

This commit is contained in:
carbotaniuman 2021-05-21 15:23:49 +00:00
parent f62eb0db00
commit b0dfc9ff12
8 changed files with 369 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

528
docker/grafana/dashboards/mangadex_at_home.json
View File

@ -14,10 +14,10 @@
}
]
},
"editable": false,
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1612974883967,
"iteration": 1617066952719,
"links": [],
"panels": [
{
@ -115,7 +115,7 @@
},
"gridPos": {
"h": 8,
"w": 8,
"w": 6,
"x": 0,
"y": 1
},
@ -250,8 +250,8 @@
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"w": 6,
"x": 6,
"y": 1
},
"id": 19,
@ -310,6 +310,167 @@
"title": "RAM",
"type": "timeseries"
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"description": "Cache size allocated, does not include metadata or other overhead, but does somewhat account for block sizes.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "hue",
"hideFrom": {
"graph": false,
"legend": false,
"tooltip": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Max"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Used"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Max"
},
"properties": [
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.lineWidth",
"value": 2
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 1
},
"id": 57,
"interval": null,
"links": [],
"options": {
"graph": {},
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltipOptions": {
"mode": "multi"
}
},
"pluginVersion": "7.4.0",
"targets": [
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(cache_used_bytes)",
"hide": false,
"interval": "",
"legendFormat": "Used",
"refId": "A",
"units": "none",
"valueHandler": "Number Threshold"
},
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(cache_max_bytes)",
"hide": false,
"instant": false,
"interval": "",
"legendFormat": "Max",
"refId": "B",
"units": "none",
"valueHandler": "Number Threshold"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Cache",
"type": "timeseries"
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
@ -408,8 +569,8 @@
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"w": 6,
"x": 18,
"y": 1
},
"id": 20,
@ -1792,7 +1953,7 @@
"type": "timeseries"
},
{
"collapsed": true,
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
@ -1801,181 +1962,180 @@
"y": 27
},
"id": 53,
"panels": [
{
"circleMaxSize": "10",
"circleMinSize": "1",
"colors": [
"#73BF69",
"#FADE2A",
"#C4162A"
],
"datasource": "Prometheus",
"decimals": 0,
"esMetric": "Count",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 11,
"x": 0,
"y": 28
},
"hideEmpty": true,
"hideZero": true,
"id": 39,
"initialZoom": "2",
"locationData": "countries",
"mapCenter": "custom",
"mapCenterLatitude": "30",
"mapCenterLongitude": "20",
"maxDataPoints": 1,
"mouseWheelZoom": false,
"pluginVersion": "7.3.4",
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"labelField": "country",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"expr": "sum(increase(requests_country_counts_total[$__range])) by (country)",
"format": "time_series",
"instant": true,
"interval": "",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": "1000,10000",
"timeFrom": null,
"timeShift": null,
"title": "Origin of requests over timerange",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"unitSingular": "",
"valueName": "current"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {},
"thresholds": {
"mode": "absolute",
"steps": []
}
},
"overrides": []
},
"fill": 8,
"fillGradient": 0,
"gridPos": {
"h": 15,
"w": 13,
"x": 11,
"y": 28
},
"hiddenSeries": false,
"id": 41,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 2,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": false
},
"percentage": false,
"pluginVersion": "7.4.0",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "topk(5, sum(rate(requests_country_counts_total[$itvl])) by (country))",
"interval": "$itvl",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Country spread [$itvl]",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:104",
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:105",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"panels": [],
"title": "GeoIP",
"type": "row"
},
{
"circleMaxSize": "10",
"circleMinSize": "1",
"colors": [
"#73BF69",
"#FADE2A",
"#C4162A"
],
"datasource": "Prometheus",
"decimals": 0,
"esMetric": "Count",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 11,
"x": 0,
"y": 28
},
"hideEmpty": true,
"hideZero": true,
"id": 39,
"initialZoom": "2",
"locationData": "countries",
"mapCenter": "custom",
"mapCenterLatitude": "30",
"mapCenterLongitude": "20",
"maxDataPoints": 1,
"mouseWheelZoom": false,
"pluginVersion": "7.3.4",
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"labelField": "country",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"expr": "sum(increase(requests_country_counts_total[$__range])) by (country)",
"format": "time_series",
"instant": true,
"interval": "",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": "1000,10000",
"timeFrom": null,
"timeShift": null,
"title": "Origin of requests over timerange",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"unitSingular": "",
"valueName": "current"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {},
"thresholds": {
"mode": "absolute",
"steps": []
}
},
"overrides": []
},
"fill": 8,
"fillGradient": 0,
"gridPos": {
"h": 15,
"w": 13,
"x": 11,
"y": 28
},
"hiddenSeries": false,
"id": 41,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 2,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": false
},
"percentage": false,
"pluginVersion": "7.4.0",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "topk(5, sum(rate(requests_country_counts_total[$itvl])) by (country))",
"interval": "$itvl",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Country spread [$itvl]",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:104",
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:105",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "30s",
@ -2069,12 +2229,12 @@
]
},
"time": {
"from": "now-30m",
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "MangaDex@Home - Personal client dashboard",
"uid": "a7sZAw2Mk",
"version": 4
}
"version": 2
}

16
src/main/kotlin/mdnet/ServerManager.kt
View File

@ -18,6 +18,8 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet
import io.micrometer.core.instrument.Gauge
import io.micrometer.core.instrument.binder.BaseUnits
import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage
@ -105,7 +107,19 @@ class ServerManager(
fun start() {
LOGGER.info { "Image server starting" }
DefaultMicrometerMetrics(registry, storage.cacheDirectory)
DefaultMicrometerMetrics(registry)
Gauge.builder(
"cache.used",
storage,
{ it.size.toDouble() }
).baseUnit(BaseUnits.BYTES).register(registry)
Gauge.builder(
"cache.max",
storage,
{ it.maxSize.toDouble() }
).baseUnit(BaseUnits.BYTES).register(registry)
loginAndStartServer()
var lastBytesSent = statistics.bytesSent.get()

2
src/main/kotlin/mdnet/cache/ImageStorage.kt vendored
View File

@ -55,7 +55,7 @@ data class Image(val data: ImageMetadata, val stream: InputStream)
*/
class ImageStorage(
var maxSize: Long,
val cacheDirectory: Path,
private val cacheDirectory: Path,
private val database: Database,
autoPrune: Boolean = true
) {

2
src/main/kotlin/mdnet/data/Token.kt
View File

@ -23,4 +23,4 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming
import java.time.OffsetDateTime
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String? = null, val ip: String? = null)
data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String)

5
src/main/kotlin/mdnet/metrics/DefaultMicrometerMetrics.kt
View File

@ -19,7 +19,6 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.metrics
import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.binder.jvm.DiskSpaceMetrics
import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics
import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics
import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
@ -30,9 +29,8 @@ import io.micrometer.core.instrument.binder.system.ProcessorMetrics
import io.micrometer.core.instrument.binder.system.UptimeMetrics
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.BuildInfo
import java.nio.file.Path
class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory: Path) {
class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry) {
init {
UptimeMetrics(
mutableListOf(
@ -47,6 +45,5 @@ class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory
JvmHeapPressureMetrics().bindTo(registry)
FileDescriptorMetrics().bindTo(registry)
LogbackMetrics().bindTo(registry)
DiskSpaceMetrics(cacheDirectory.toFile()).bindTo(registry)
}
}

2
src/main/kotlin/mdnet/server/ImageHandler.kt
View File

@ -44,7 +44,7 @@ class ImageServer(
registry: PrometheusMeterRegistry
) {
private val executor = Executors.newCachedThreadPool()
private val cacheLookupTimer = Timer.builder("cache_lookup")
private val cacheLookupTimer = Timer.builder("cache.lookup")
.publishPercentiles(0.5, 0.75, 0.9, 0.99)
.register(registry)

5
src/main/kotlin/mdnet/server/ImageServer.kt
View File

@ -22,6 +22,7 @@ import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry
import io.github.resilience4j.micrometer.tagged.TaggedCircuitBreakerMetrics
import io.micrometer.core.instrument.FunctionCounter
import io.micrometer.core.instrument.binder.BaseUnits
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
@ -98,10 +99,10 @@ fun getServer(
)
FunctionCounter.builder(
"client_sent_bytes",
"client.sent",
statistics,
{ it.bytesSent.get().toDouble() }
).register(registry)
).baseUnit(BaseUnits.BYTES).register(registry)
val verifier = TokenVerifier(
tokenKey = remoteSettings.tokenKey,

3
src/main/kotlin/mdnet/settings/PingResult.kt
View File

@ -36,6 +36,7 @@ data class RemoteSettings(
val imageServer: Uri,
val latestBuild: Int,
val url: Uri,
val clientId: String,
@field:Secret val tokenKey: ByteArray,
val compromised: Boolean,
val paused: Boolean,
@ -51,6 +52,7 @@ data class RemoteSettings(
if (imageServer != other.imageServer) return false
if (latestBuild != other.latestBuild) return false
if (url != other.url) return false
if (clientId != other.clientId) return false
if (!tokenKey.contentEquals(other.tokenKey)) return false
if (compromised != other.compromised) return false
if (paused != other.paused) return false
@ -64,6 +66,7 @@ data class RemoteSettings(
var result = imageServer.hashCode()
result = 31 * result + latestBuild
result = 31 * result + url.hashCode()
result = 31 * result + clientId.hashCode()
result = 31 * result + tokenKey.contentHashCode()
result = 31 * result + compromised.hashCode()
result = 31 * result + paused.hashCode()