Merge branch 'fix/threads-global' into 'master'

Made Netty thread count global instead of per-cpu

See merge request mangadex/mangadex_at_home!18
This commit is contained in:
Amos Ng 2020-06-12 14:17:38 +00:00
commit 32a2fecfd0
No known key found for this signature in database
GPG key ID: 89086414F634D123
4 changed files with 25 additions and 11 deletions

View file

@ -7,10 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- [2020-06-12] Added CHANGELOG.md by [@lflare].
- [2020-06-12] Added on-read image migrator to 4-deep subdirectory format by [@lflare].
### Changed
- [2020-06-12] Raised ApacheClient socket limit to `2**18` by [@lflare].
- [2020-06-12] Changed gradle versioning to using `git describe` by [@lflare].
- [2020-06-12] Made Netty thread count global instead of per-cpu by [@lflare].
- [2020-06-12] Store cache files in a 4-deep subdirectory to improve performance by [@lflare].
### Deprecated

View file

@ -16,17 +16,17 @@ public final class ClientSettings {
private final int clientPort;
@SerializedName("client_secret")
private final String clientSecret;
@SerializedName("threads_per_cpu")
private final int threadsPerCpu;
@SerializedName("threads")
private final int threads;
public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond,
int clientPort, String clientSecret, int threadsPerCpu) {
int clientPort, String clientSecret, int threads) {
this.maxCacheSizeMib = maxCacheSizeMib;
this.maxBandwidthMibPerHour = maxBandwidthMibPerHour;
this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond;
this.clientPort = clientPort;
this.clientSecret = Objects.requireNonNull(clientSecret);
this.threadsPerCpu = threadsPerCpu;
this.threads = threads;
}
public long getMaxCacheSizeMib() {
@ -49,15 +49,15 @@ public final class ClientSettings {
return clientSecret;
}
public int getThreadsPerCpu() {
return (threadsPerCpu > 0) ? threadsPerCpu : 16;
public int getThreads() {
return (threads > 0) ? threads : 16;
}
@Override
public String toString() {
return "ClientSettings{" + "maxCacheSizeMib=" + maxCacheSizeMib + ", maxBandwidthMibPerHour="
+ maxBandwidthMibPerHour + ", maxBurstRateKibPerSecond=" + maxBurstRateKibPerSecond + ", clientPort="
+ clientPort + ", clientSecret='" + "<hidden>" + '\'' + ", threadsPerCpu=" + getThreadsPerCpu() + '}';
+ clientPort + ", clientSecret='" + "<hidden>" + '\'' + ", threads=" + getThreads() + '}';
}
public static boolean isSecretValid(String clientSecret) {

View file

@ -945,6 +945,9 @@ public final class DiskLruCache implements Closeable {
/** Lengths of this entry's files. */
private final long[] lengths;
/** Subkey pathing for cache files. */
private final String subkeypath;
/** True if this entry has ever been published. */
private boolean readable;
@ -957,6 +960,9 @@ public final class DiskLruCache implements Closeable {
private Entry(String key) {
this.key = key;
this.lengths = new long[valueCount];
// Splits the keys into a list of two characters, and join it together to use it for sub-directorying
this.subkeypath = File.separator + String.join(File.separator, key.substring(0, 8).replaceAll("..(?!$)", "$0 ").split(" "));
}
public String getLengths() {
@ -987,11 +993,17 @@ public final class DiskLruCache implements Closeable {
}
public File getCleanFile(int i) {
return new File(directory, key + "." + i);
// Move files to new caching tree if exists
File old_cache = new File(directory, key + "." + i);
File new_cache = new File(directory + subkeypath, key + "." + i);
if (old_cache.exists()) {
old_cache.renameTo(new_cache);
}
return new_cache;
}
public File getDirtyFile(int i) {
return new File(directory, key + "." + i + ".tmp");
return new File(directory + subkeypath, key + "." + i + ".tmp");
}
}
}

View file

@ -37,8 +37,7 @@ import javax.net.ssl.SSLException
private val LOGGER = LoggerFactory.getLogger("Application")
class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings: ClientSettings, private val stats: AtomicReference<Statistics>) : ServerConfig {
private val threadsToAllocate: Int
get() = Runtime.getRuntime().availableProcessors() * clientSettings.threadsPerCpu
private val threadsToAllocate = clientSettings.getThreads()
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
private val masterGroup = NioEventLoopGroup(threadsToAllocate)