diff --git a/README.md b/README.md index e460b080..3c0e7e08 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ To run a Renaissance benchmark, you need to have a JRE version 11 (or later) installed and execute the following `java` command: ``` -$ java -jar 'renaissance-gpl-0.15.0.jar' +$ java -jar 'renaissance-gpl-0.16.0.jar' ``` In the above command, `` is the list of benchmarks that you want to run. @@ -143,6 +143,12 @@ The following is the complete list of benchmarks, separated into groups. \ Default repetitions: 50; GPL2 license, GPL3 distribution; Supported JVM: 11 and later +#### kotlin + +- `http4k` - Runs the http4k server and tests the throughput of the server by sending requests to the server. + \ + Default repetitions: 20; APACHE2 license, MIT distribution; Supported JVM: 11 and later + #### scala - `dotty` - Runs the Dotty compiler on a set of source code files. @@ -268,7 +274,7 @@ arguments to that plugin (or policy). The following is a complete list of command-line options. ``` -Renaissance Benchmark Suite, version 0.15.0 +Renaissance Benchmark Suite, version 0.16.0 Usage: renaissance [options] [benchmark-specification] -h, --help Prints this usage text. @@ -315,7 +321,7 @@ $ tools/sbt/bin/sbt renaissanceJmhPackage To run the benchmarks using JMH, you can execute the following `java` command: ``` -$ java -jar 'renaissance-jmh/target/renaissance-jmh-0.15.0.jar' +$ java -jar 'renaissance-jmh/target/renaissance-jmh-0.16.0.jar' ``` diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/Http4kBenchmark.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/Http4kBenchmark.kt new file mode 100644 index 00000000..cf4c88f1 --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/Http4kBenchmark.kt @@ -0,0 +1,118 @@ +package org.renaissance.http4k + +import kotlinx.coroutines.runBlocking +import org.http4k.client.OkHttp +import org.renaissance.Benchmark +import org.renaissance.Benchmark.* +import org.renaissance.BenchmarkContext +import org.renaissance.BenchmarkResult +import org.renaissance.BenchmarkResult.Validators +import org.renaissance.License +import org.renaissance.http4k.workload.WorkloadClient +import org.renaissance.http4k.workload.WorkloadConfiguration +import org.renaissance.http4k.workload.WorkloadServer + +@Name("http4k") +@Group("kotlin") +@Summary("Runs the http4k server and tests the throughput of the server by sending requests to the server.") +@Licenses(License.APACHE2) +@Repetitions(20) +@Parameter( + name = "host", + defaultValue = "localhost", + summary = "Host of the server." +) +@Parameter( + name = "port", + defaultValue = "0", + summary = "Port of the server." +) +@Parameter( + name = "read_workload_repeat_count", + defaultValue = "5", + summary = "Number of read requests to generate." +) +@Parameter( + name = "write_workload_repeat_count", + defaultValue = "5", + summary = "Number of write requests to generate." +) +@Parameter( + name = "ddos_workload_repeat_count", + defaultValue = "5", + summary = "Number of ddos requests to generate." +) +@Parameter( + name = "mixed_workload_repeat_count", + defaultValue = "5", + summary = "Number of mixed requests to generate." +) +@Parameter( + name = "workload_count", + defaultValue = "450", + summary = "Number of workloads to generate. Each workload consists of read, write, ddos and mixed requests." +) +@Parameter( + name = "max_threads", + defaultValue = "\$cpu.count", + summary = "Maximum number of threads to use for the executor of the requests." +) +@Parameter( + name = "workload_selection_seed", + defaultValue = "42", + summary = "Seed used to generate random workloads." +) +@Configuration( + name = "test", + settings = [ + "max_threads = 2", + "workload_count = 100", + ] +) +@Configuration(name = "jmh") +class Http4kBenchmark : Benchmark { + private lateinit var server: WorkloadServer + private lateinit var client: WorkloadClient + private lateinit var configuration: WorkloadConfiguration + + override fun run(context: BenchmarkContext): BenchmarkResult = runBlocking { + val workloadSummary = client.workload() + Validators.simple("Workload count", configuration.workloadCount.toLong(), workloadSummary.workloadCount) + } + + override fun setUpBeforeEach(context: BenchmarkContext) { + configuration = context.toWorkloadConfiguration() + server = configuration.toWorkloadServer() + server.start() + + // If port value is 0, server allocates an empty port which has to be saved to allow client requests. + configuration = configuration.copy(port = server.port()) + client = configuration.toWorkloadClient() + } + + override fun tearDownAfterEach(context: BenchmarkContext) { + server.stop() + } + + private fun BenchmarkContext.toWorkloadConfiguration(): WorkloadConfiguration = WorkloadConfiguration( + host = parameter("host").value(), + port = parameter("port").value().toInt(), + readWorkloadRepeatCount = parameter("read_workload_repeat_count").value().toInt(), + writeWorkloadRepeatCount = parameter("write_workload_repeat_count").value().toInt(), + ddosWorkloadRepeatCount = parameter("ddos_workload_repeat_count").value().toInt(), + mixedWorkloadRepeatCount = parameter("mixed_workload_repeat_count").value().toInt(), + workloadCount = parameter("workload_count").value().toInt(), + maxThreads = parameter("max_threads").value().toInt(), + workloadSelectionSeed = parameter("workload_selection_seed").value().toLong() + ) + + private fun WorkloadConfiguration.toWorkloadClient(): WorkloadClient = + WorkloadClient(OkHttp(), this) + + private fun WorkloadConfiguration.toWorkloadServer(): WorkloadServer = + WorkloadServer(port) +} + + + + diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/model/Product.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/model/Product.kt new file mode 100644 index 00000000..e13c7276 --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/model/Product.kt @@ -0,0 +1,11 @@ +package org.renaissance.http4k.model + +import org.http4k.core.Body +import org.http4k.format.Moshi.auto + +internal data class Product(val id: String, val name: String) { + internal companion object { + internal val productLens = Body.auto().toLens() + internal val productsLens = Body.auto>().toLens() + } +} \ No newline at end of file diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadClient.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadClient.kt new file mode 100644 index 00000000..1a390c8b --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadClient.kt @@ -0,0 +1,145 @@ +package org.renaissance.http4k.workload + +import kotlinx.coroutines.* +import org.http4k.core.HttpHandler +import org.http4k.core.Method +import org.http4k.core.Request +import org.renaissance.http4k.model.Product +import java.util.* +import java.util.concurrent.atomic.AtomicLong +import kotlin.random.Random + +/** + * Client used to generate workloads for the http4k server. + * The client sends requests to the server based on the workload type. + * @param client HttpHandler used to send requests to the server. + * @param configuration WorkloadConfiguration used to generate the workload. + */ +internal class WorkloadClient( + private val client: HttpHandler, private val configuration: WorkloadConfiguration +) { + private val getProductsCounter = AtomicLong(0) + private val getProductCounter = AtomicLong(0) + private val postProductCounter = AtomicLong(0) + + private val readCounter = AtomicLong(0) + private val writeCounter = AtomicLong(0) + private val ddosCounter = AtomicLong(0) + private val mixedCounter = AtomicLong(0) + + private val workloadCounter = AtomicLong(0) + + private val dispatcher = Dispatchers.IO.limitedParallelism(configuration.maxThreads, "Workload") + + /** + * Starts the workload on the server based on [configuration]. + * Each workload consists of read, write, ddos and mixed requests. + * The number of workloads is determined by [WorkloadConfiguration.workloadCount]. + * The number of requests for each workload type is determined by the corresponding configuration value. + * Random workload is generated for each iteration based on the seed in [WorkloadConfiguration.workloadSelectionSeed]. + * @return WorkloadResult containing number of requests per type used for validation. + */ + suspend fun workload(): WorkloadSummary = coroutineScope { + val random = Random(configuration.workloadSelectionSeed) + withContext(dispatcher) { + range(configuration.workloadCount).flatMap { + when (random.nextWorkload()) { + WorkloadType.READ -> range(configuration.readWorkloadRepeatCount).map { async { client.readWorkload() } } + WorkloadType.WRITE -> range(configuration.writeWorkloadRepeatCount).map { async { client.writeWorkload() } } + WorkloadType.DDOS -> range(configuration.ddosWorkloadRepeatCount).map { async { client.ddosWorkload() } } + WorkloadType.MIXED -> range(configuration.mixedWorkloadRepeatCount).map { async { client.mixedWorkload() } } + }.also { workloadCounter.incrementAndGet() } + }.awaitAll() + + WorkloadSummary( + getProductsCount = getProductsCounter.get(), + getProductCount = getProductCounter.get(), + postProductCount = postProductCounter.get(), + readCount = readCounter.get(), + writeCount = writeCounter.get(), + ddosCount = ddosCounter.get(), + mixedCount = mixedCounter.get(), + workloadCount = workloadCounter.get() + ) + } + } + + /** + * Read workload gets all products and then iterates over each one and gets the specific product. + */ + private fun HttpHandler.readWorkload() { + val products = getProducts() + products.forEach { product -> + getProduct(product.id) + } + readCounter.incrementAndGet() + } + + /** + * Write workload creates a new product. + */ + private fun HttpHandler.writeWorkload() { + val product = generateProduct() + postProduct(product) + writeCounter.incrementAndGet() + } + + /** + * DDOS workload reads all products 10 times in a row. + */ + private fun HttpHandler.ddosWorkload() { + repeat(10) { + getProducts() + } + ddosCounter.incrementAndGet() + } + + /** + * Mixed workload reads all products, then creates a new product and fetches it afterward. + */ + private fun HttpHandler.mixedWorkload() { + getProducts() + val product = generateProduct() + postProduct(product) + getProduct(product.id) + mixedCounter.incrementAndGet() + } + + /** + * Helper functions to interact with the server. + */ + private fun HttpHandler.getProducts(): List = + Product.productsLens(this(Request(Method.GET, configuration.url("product")))).toList() + .also { getProductsCounter.incrementAndGet() } + + private fun HttpHandler.getProduct(id: String) = + this(Request(Method.GET, configuration.url("product/$id"))).also { getProductCounter.incrementAndGet() } + + private fun HttpHandler.postProduct(product: Product) = this( + Product.productLens( + product, + Request(Method.POST, configuration.url("product")) + ) + ).also { postProductCounter.incrementAndGet() } + + /** + * Helper function to generate a URL from the configuration. + */ + private fun WorkloadConfiguration.url(endpoint: String) = "http://$host:$port/$endpoint" + + /** + * Helper function to generate a random workload type. + */ + private fun Random.nextWorkload() = WorkloadType.entries[nextInt(WorkloadType.entries.size)] + + /** + * Helper function to generate a new product with random id. + */ + private fun generateProduct(): Product { + val id = UUID.randomUUID().toString() + val name = "Product $id" + return Product(id, name) + } + + private fun range(end: Int) = (1..end) +} \ No newline at end of file diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadConfiguration.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadConfiguration.kt new file mode 100644 index 00000000..bfe198d1 --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadConfiguration.kt @@ -0,0 +1,13 @@ +package org.renaissance.http4k.workload + +internal data class WorkloadConfiguration( + val host: String, + val port: Int, + val readWorkloadRepeatCount: Int, + val writeWorkloadRepeatCount: Int, + val ddosWorkloadRepeatCount: Int, + val mixedWorkloadRepeatCount: Int, + val workloadCount: Int, + val maxThreads: Int, + val workloadSelectionSeed: Long, +) \ No newline at end of file diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadServer.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadServer.kt new file mode 100644 index 00000000..a2216397 --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadServer.kt @@ -0,0 +1,51 @@ +package org.renaissance.http4k.workload + +import org.http4k.core.HttpHandler +import org.http4k.core.Method +import org.http4k.core.Response +import org.http4k.core.Status +import org.http4k.routing.bind +import org.http4k.routing.path +import org.http4k.routing.routes +import org.http4k.server.Http4kServer +import org.http4k.server.Undertow +import org.http4k.server.asServer +import org.renaissance.http4k.model.Product +import java.util.concurrent.ConcurrentHashMap + +internal class WorkloadServer(port: Int) : Http4kServer { + private val server = app().asServer(Undertow(port)) + private val products: MutableMap = ConcurrentHashMap() + + private fun app(): HttpHandler = routes( + "/product" bind Method.GET to { Product.productsLens(products.values.toTypedArray(), Response(Status.OK)) }, + "/product/{id}" bind Method.GET to { + when (val id = it.path("id")) { + null -> Response(Status.BAD_REQUEST) + !in products -> Response(Status.NOT_FOUND) + else -> { + val product = products[id] ?: error("Invariant error: Product $it should be present") + Product.productLens(product, Response(Status.OK)) + } + } + }, + "/product" bind Method.POST to { + val product = Product.productLens(it) + products[product.id] = product + Response(Status.CREATED) + } + ) + + override fun port(): Int = server.port() + + override fun start(): Http4kServer { + server.start() + return this + } + + override fun stop(): Http4kServer { + server.stop() + products.clear() + return this + } +} \ No newline at end of file diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadSummary.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadSummary.kt new file mode 100644 index 00000000..8502e142 --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadSummary.kt @@ -0,0 +1,12 @@ +package org.renaissance.http4k.workload + +internal data class WorkloadSummary( + val getProductsCount: Long, + val getProductCount: Long, + val postProductCount: Long, + val readCount: Long, + val writeCount: Long, + val ddosCount: Long, + val mixedCount: Long, + val workloadCount: Long +) \ No newline at end of file diff --git a/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadType.kt b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadType.kt new file mode 100644 index 00000000..d4e520ab --- /dev/null +++ b/benchmarks/http4k/src/main/kotlin/org/renaissance/http4k/workload/WorkloadType.kt @@ -0,0 +1,8 @@ +package org.renaissance.http4k.workload + +internal enum class WorkloadType { + READ, + WRITE, + DDOS, + MIXED +} \ No newline at end of file diff --git a/build.sbt b/build.sbt index b24893d0..44ee526f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,4 @@ +import kotlin.Keys.{kotlinVersion, kotlincJvmTarget} import org.renaissance.License import sbt.Def import sbt.Package @@ -353,6 +354,23 @@ lazy val jdkStreamsBenchmarks = (project in file("benchmarks/jdk-streams")) ) .dependsOn(renaissanceCore % "provided") +lazy val http4kBenchmarks = (project in file("benchmarks/http4k")) + .enablePlugins(KotlinPlugin) + .settings( + name := "http4k", + commonSettingsNoScala, + kotlinVersion := "2.0.0", + kotlincJvmTarget := "1.8", + libraryDependencies ++= Seq( + "org.http4k" % "http4k-core" % "5.29.0.0", + "org.http4k" % "http4k-server-undertow" % "5.29.0.0", + "org.http4k" % "http4k-client-okhttp" % "5.29.0.0", + "org.http4k" % "http4k-format-moshi" % "5.29.0.0", + "org.jetbrains.kotlinx" % "kotlinx-coroutines-core" % "1.9.0" + ) + ) + .dependsOn(renaissanceCore % "provided") + lazy val neo4jBenchmarks = (project in file("benchmarks/neo4j")) .settings( name := "neo4j", @@ -513,6 +531,7 @@ val renaissanceBenchmarks: Seq[Project] = Seq( databaseBenchmarks, jdkConcurrentBenchmarks, jdkStreamsBenchmarks, + http4kBenchmarks, neo4jBenchmarks, rxBenchmarks, scalaDottyBenchmarks, diff --git a/project/plugins.sbt b/project/plugins.sbt index 614ead4a..e58e743e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,2 +1,3 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.4") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.2") +addSbtPlugin("org.jetbrains.scala" % "sbt-kotlin-plugin" % "3.0.3")