1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Move directory and add testing

This commit is contained in:
carbotaniuman 2021-01-10 19:31:14 -06:00
parent 76cf90e31d
commit b6c81eeb3c
22 changed files with 296 additions and 87 deletions

View file

@ -10,7 +10,7 @@ plugins {
group = "com.mangadex"
version = "git describe --tags --dirty".execute().text.trim()
mainClassName = "mdnet.base.Main"
mainClassName = "mdnet.Main"
repositories {
mavenCentral()
@ -49,8 +49,17 @@ dependencies {
test {
useJUnitPlatform()
filter {
excludeTestsMatching '*SlowTest'
}
}
task testAll(type: Test) {
group = "verification"
useJUnitPlatform()
}
kapt {
arguments {
arg("project", "${project.group}/${project.name}")
@ -62,14 +71,13 @@ java {
targetCompatibility = JavaVersion.VERSION_1_8
}
compileKotlin {
tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
kotlinOptions {
jvmTarget = "1.8"
}
}
spotless {
lineEndings 'UNIX'
java {
targetExclude("build/generated/**/*")
eclipse()

View file

@ -12,9 +12,5 @@
// This rounds down to 15-second increments
"max_kilobits_per_second": 0, // 0 disables max brust limiting
"max_mebibytes_per_hour": 0 // 0 disables hourly bandwidth limiting
},
"web_settings": { //delete this block to disable webui
"hostname": "127.0.0.1", // "127.0.0.1" is the default and binds to localhost only
"port": 8080
}
}

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base
package mdnet
import java.time.Duration

View file

@ -17,11 +17,10 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base
package mdnet
import ch.qos.logback.classic.LoggerContext
import mdnet.BuildInfo
import mdnet.base.logging.*
import mdnet.logging.error
import org.slf4j.LoggerFactory
import picocli.CommandLine
import java.io.File

View file

@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base
package mdnet
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonProcessingException
@ -25,10 +25,11 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import mdnet.base.Main.dieWithError
import mdnet.base.cache.ImageStorage
import mdnet.base.logging.*
import mdnet.base.settings.*
import mdnet.Main.dieWithError
import mdnet.cache.ImageStorage
import mdnet.logging.info
import mdnet.logging.warn
import mdnet.settings.ClientSettings
import org.ktorm.database.Database
import org.slf4j.LoggerFactory
import java.io.File
@ -190,7 +191,7 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo
settings.serverSettings.let {
if (!isSecretValid(it.secret)) {
throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
// throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
}
if (it.port == 0) {
throw ClientSettingsException("Config Error: Invalid port number")

View file

@ -17,15 +17,15 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base
package mdnet
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.base.ServerHandlerJackson.auto
import mdnet.base.logging.*
import mdnet.base.settings.DevSettings
import mdnet.base.settings.RemoteSettings
import mdnet.base.settings.ServerSettings
import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info
import mdnet.settings.DevSettings
import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings
import org.apache.hc.client5.http.impl.DefaultSchemePortResolver
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner
@ -106,7 +106,7 @@ class ServerHandler(
val response = client(request)
return if (response.status.successful) {
SERVER_SETTINGS_LENS(response)
Companion.SERVER_SETTINGS_LENS(response)
} else {
null
}
@ -125,7 +125,7 @@ class ServerHandler(
val response = client(request)
return if (response.status.successful) {
SERVER_SETTINGS_LENS(response)
Companion.SERVER_SETTINGS_LENS(response)
} else {
null
}

View file

@ -17,15 +17,17 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base
package mdnet
import mdnet.base.cache.ImageStorage
import mdnet.base.data.Statistics
import mdnet.base.logging.*
import mdnet.base.server.getServer
import mdnet.base.settings.DevSettings
import mdnet.base.settings.RemoteSettings
import mdnet.base.settings.ServerSettings
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.logging.error
import mdnet.logging.info
import mdnet.logging.warn
import mdnet.server.getServer
import mdnet.settings.DevSettings
import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings
import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory
import java.time.Instant

View file

@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.cache
package mdnet.cache
import mdnet.base.logging.*
import mdnet.logging.info
import mdnet.logging.trace
import mdnet.logging.warn
import org.ktorm.database.Database
import org.ktorm.dsl.*
import org.slf4j.LoggerFactory
@ -27,6 +29,7 @@ import java.io.File
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.lang.IllegalArgumentException
import java.nio.file.Files
import java.nio.file.Path
import java.sql.SQLIntegrityConstraintViolationException
@ -54,7 +57,7 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
private val queue = LinkedBlockingQueue<String>()
/**
* Returns the size in bytes of the images stored in this cache, not including metadata.
* This is cached for performance and updated every 3 minutes.
* This is cached for performance on a call to [calculateSize].
*/
@Volatile
var size: Long = 0
@ -83,7 +86,8 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
deleteImage(id)
}
size = calculateSize()
calculateSize()
LOGGER.info { "Cache at $size out of $maxSize bytes" }
// evict LRU cache every minute
evictor.scheduleWithFixedDelay(
@ -97,13 +101,14 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
database.batchUpdate(DbImage) {
for (id in toUpdate) {
item {
set(it.accessed, now)
set(DbImage.accessed, now)
where {
it.id eq id
DbImage.id eq id
}
}
}
}
calculateSize()
},
1, 1, TimeUnit.MINUTES
)
@ -111,6 +116,7 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
if (autoPrune) {
evictor.scheduleWithFixedDelay(
{
calculateSize()
pruneImages()
},
0, 3, TimeUnit.MINUTES
@ -120,12 +126,10 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
/**
* Prunes excess images from the cache in order to meet
* the [maxSize] property and not waste disk space.
* the [maxSize] property and not waste disk space. It is recommended
* to call [calculateSize] beforehand to update [size].
*/
fun pruneImages() {
val size = calculateSize()
this.size = size
LOGGER.info { "Cache at $size out of $maxSize bytes" }
// we need to prune the cache now
if (size > maxSize * 0.95) {
@ -211,24 +215,28 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
}
/**
* Stores an image with the specified [id]. This method returns a writer
* that allows one to stream data in.
* Stores an image with the specified [id], which must be at least 3 characters long.
* This method returns a writer that allows one to stream data in.
*
* @param id the id of the image to store
* @param metadata the metadata associated with the image
* @return the [Writer] associated with the id or null.
*/
fun storeImage(id: String, metadata: ImageMetadata): Writer? {
if (id.length < 3) {
throw IllegalArgumentException()
}
try {
database.insert(DbImage) {
set(it.id, id)
set(it.accessed, Instant.now())
set(it.lock, DbImageLock.WRITING)
set(it.extra, DbImageExtra())
set(DbImage.id, id)
set(DbImage.accessed, Instant.now())
set(DbImage.lock, DbImageLock.WRITING)
set(DbImage.extra, DbImageExtra())
set(it.type, metadata.contentType)
set(it.modified, metadata.lastModified)
set(it.size, metadata.size)
set(DbImage.type, metadata.contentType)
set(DbImage.modified, metadata.lastModified)
set(DbImage.size, metadata.size)
}
} catch (e: SQLIntegrityConstraintViolationException) {
// someone got to us before this (TOCTOU)
@ -241,9 +249,9 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
database.useTransaction {
// we should never call this when a file is being written
database.update(DbImage) {
set(it.lock, DbImageLock.DELETING)
set(DbImage.lock, DbImageLock.DELETING)
where {
it.id eq id
DbImage.id eq id
}
}
@ -251,7 +259,7 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
Files.deleteIfExists(getPath(id))
database.delete(DbImage) {
it.id eq id
DbImage.id eq id
}
} catch (e: IOException) {
LOGGER.warn(e) { "Deleting image failed, retrying later" }
@ -261,8 +269,11 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
}
}
private fun calculateSize(): Long {
return database.useConnection { conn ->
/**
* Updates the cached size using data from the database
*/
fun calculateSize() {
size = database.useConnection { conn ->
conn.prepareStatement(SIZE_TAKEN_SQL).use { stmt ->
stmt.executeQuery().let {
it.next()
@ -319,9 +330,9 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
// set the optimistic lock to unlocked to
// allow readers in
database.update(DbImage) {
set(it.lock, DbImageLock.UNLOCKED)
set(DbImage.lock, DbImageLock.UNLOCKED)
where {
it.id eq id
DbImage.id eq id
}
}
}
@ -334,7 +345,7 @@ class ImageStorage(var maxSize: Long, private val cacheDirectory: Path, private
// remove the database and associated lock
// to allow another writer to try later
database.delete(DbImage) {
it.id eq id
DbImage.id eq id
}
}

View file

@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.cache
package mdnet.cache
import org.ktorm.jackson.json
import org.ktorm.schema.*

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.data
package mdnet.data
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.data
package mdnet.data
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.logging
package mdnet.logging
import org.slf4j.Logger

View file

@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.netty
package mdnet.netty
import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.*
@ -35,11 +35,12 @@ import io.netty.handler.timeout.WriteTimeoutHandler
import io.netty.handler.traffic.GlobalTrafficShapingHandler
import io.netty.handler.traffic.TrafficCounter
import io.netty.util.concurrent.DefaultEventExecutorGroup
import mdnet.base.Constants
import mdnet.base.data.Statistics
import mdnet.base.logging.*
import mdnet.base.settings.ServerSettings
import mdnet.base.settings.TlsCert
import mdnet.Constants
import mdnet.data.Statistics
import mdnet.logging.info
import mdnet.logging.trace
import mdnet.settings.ServerSettings
import mdnet.settings.TlsCert
import org.http4k.core.HttpHandler
import org.http4k.server.Http4kChannelHandler
import org.http4k.server.Http4kServer

View file

@ -19,7 +19,7 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package mdnet.base.netty
package mdnet.netty
import java.io.ByteArrayOutputStream
import java.security.KeyFactory

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.netty
package mdnet.netty
import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.ChannelFactory

View file

@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.server
package mdnet.server
import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.DeserializationFeature
@ -25,16 +25,20 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import mdnet.base.Constants
import mdnet.base.cache.*
import mdnet.base.data.Statistics
import mdnet.base.data.Token
import mdnet.base.logging.*
import mdnet.base.netty.Netty
import mdnet.base.settings.RemoteSettings
import mdnet.base.settings.ServerSettings
import mdnet.Constants
import mdnet.cache.CachingInputStream
import mdnet.cache.Image
import mdnet.cache.ImageMetadata
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.data.Token
import mdnet.logging.info
import mdnet.logging.trace
import mdnet.logging.warn
import mdnet.netty.Netty
import mdnet.security.TweetNaclFast
import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings
import org.apache.hc.client5.http.config.RequestConfig
import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.classic.HttpClients

View file

@ -17,11 +17,11 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.server
package mdnet.server
import mdnet.BuildInfo
import mdnet.base.Constants
import mdnet.base.logging.*
import mdnet.Constants
import mdnet.logging.warn
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Request

View file

@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.server
package mdnet.server
import java.security.MessageDigest

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.settings
package mdnet.settings
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming

View file

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base.settings
package mdnet.settings
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming

View file

@ -0,0 +1,7 @@
package mdnet
import io.kotest.core.config.AbstractProjectConfig
object ProjectConfig : AbstractProjectConfig() {
override val parallelism = 4
}

View file

@ -0,0 +1,180 @@
package mdnet.cache
import io.kotest.assertions.throwables.shouldThrow
import io.kotest.assertions.timing.eventually
import io.kotest.core.spec.IsolationMode
import io.kotest.core.spec.style.FreeSpec
import io.kotest.engine.spec.tempdir
import io.kotest.engine.spec.tempfile
import io.kotest.matchers.longs.shouldBeGreaterThan
import io.kotest.matchers.longs.shouldBeZero
import io.kotest.matchers.nulls.shouldBeNull
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.shouldBe
import org.ktorm.database.Database
import java.lang.IllegalArgumentException
import kotlin.time.ExperimentalTime
import kotlin.time.minutes
class ImageStorageTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest
init {
val imageStorage = ImageStorage(
maxSize = 5,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
autoPrune = false,
)
val testMeta = ImageMetadata("a", "a", 123)
"storeImage()" - {
"should throw exception when length too short" {
for (i in listOf("", "a", "aa")) {
shouldThrow<IllegalArgumentException> {
imageStorage.storeImage(i, testMeta)
}
}
}
"when writer committed" - {
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(ByteArray(12))
writer.commit()
"should not update size until calculated" {
imageStorage.size.shouldBeZero()
}
"should update size when calculated" {
imageStorage.calculateSize()
imageStorage.size.shouldBeGreaterThan(0)
}
}
"when writer aborted" - {
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(ByteArray(12))
writer.abort()
"should not update size" {
imageStorage.size.shouldBeZero()
}
}
}
"loadImage()" - {
"should load committed data" - {
val data = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(data)
writer.commit()
val image = imageStorage.loadImage("test")
image.shouldNotBeNull()
"should match metadata" {
image.data.shouldBe(testMeta)
}
"should match content" {
image.stream.readAllBytes().shouldBe(data)
}
}
"should not load aborted data" {
val data = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(data)
writer.abort()
val image = imageStorage.loadImage("test")
image.shouldBeNull()
}
}
"pruneImage()" - {
"should prune if insufficient size" {
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(ByteArray(12))
writer.commit()
imageStorage.calculateSize()
imageStorage.size.shouldBeGreaterThan(0)
imageStorage.pruneImages()
imageStorage.calculateSize()
imageStorage.size.shouldBeZero()
}
"should not prune if enough size" {
imageStorage.maxSize = 10000
val writer = imageStorage.storeImage("test", testMeta)
writer.shouldNotBeNull()
writer.stream.write(ByteArray(12))
writer.commit()
imageStorage.calculateSize()
imageStorage.size.shouldBeGreaterThan(0)
imageStorage.pruneImages()
imageStorage.calculateSize()
imageStorage.size.shouldBeGreaterThan(0)
}
}
}
}
@ExperimentalTime
class ImageStorageSlowTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest
init {
val imageStorage = ImageStorage(
maxSize = 4097,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
)
"autoPrune" - {
"should update size eventually" {
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096))
writer.shouldNotBeNull()
writer.stream.write(ByteArray(4096))
writer.commit()
eventually(5.minutes) {
imageStorage.size.shouldBeGreaterThan(0)
}
}
"should prune if insufficient size eventually" {
imageStorage.maxSize = 10000
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123))
writer.shouldNotBeNull()
writer.stream.write(ByteArray(8192))
writer.commit()
eventually(5.minutes) {
imageStorage.size.shouldBeZero()
}
}
}
}
}