[GR-44732] Add new microservice benchmarks

PullRequest: graal/13993

Author: d-kozak

java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py

@@ -172,16 +172,13 @@ def parserNames(self):
         return super(TemporaryWorkdirMixin, self).parserNames() + ["temporary_workdir_parser"]
 
 
-class BaseMicroserviceBenchmarkSuite(object):
+class BaseMicroserviceBenchmarkSuite(mx_benchmark.BenchmarkSuite):
     def group(self):
         return "Graal"
 
     def subgroup(self):
         return "graal-compiler"
 
-    def version(self):
-        raise NotImplementedError()
-
     def validateReturnCode(self, retcode):
         return retcode == 143
 
@@ -238,6 +235,7 @@ def mainClass(self):
     def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         lib = self.applicationDist()
         classpath = os.path.join(lib, "BOOT-INF/classes")
+        classpath += ':' + os.path.join(lib, "classes")
         for filename in os.listdir(os.path.join(lib, "BOOT-INF/lib")):
             if filename.endswith(".jar"):
                 classpath = classpath + ":" + os.path.join(lib, "BOOT-INF/lib", filename)
@@ -258,8 +256,16 @@ def default_stages(self):
 
 
 class BasePetClinicBenchmarkSuite(BaseSpringBenchmarkSuite):
-    def version(self):
-        return "0.1.6"
+    """
+    Version 0.1.7 MIGHT NOT be fully functional. So far, it was used only to collect image build time metrics.
+    """
+    def availableSuiteVersions(self):
+        # 0.1.6 is still based on org.springframework.experimental
+        # 0.1.7 is based on Spring 3, hence with official GraalVM NI support, requires Java 17 or newer
+        return ["0.1.6", "0.1.7"]
+
+    def defaultSuiteVersion(self):
+        return self.availableSuiteVersions()[0]
 
     def applicationDist(self):
         return mx.library("PETCLINIC_" + self.version(), True).get_path(True)
@@ -477,6 +483,111 @@ def extra_image_build_argument(self, benchmark, args):
     def default_stages(self):
         return ['instrument-image', 'instrument-run', 'image', 'run']
 
+class BaseQuarkusRegistryBenchmark(BaseQuarkusBenchmarkSuite, mx_sdk_benchmark.BaseMicroserviceBenchmarkSuite):
+    """
+    This benchmark is NOT yet fully functional - there is not load for measuring its runtime performance.
+    It is only useful to collect image build time metrics.
+    """
+
+    def version(self):
+        return "0.0.1"
+
+    def name(self):
+        return "quarkus"
+
+    def benchmarkList(self, bmSuiteArgs):
+        return ["registry"]
+
+    def default_stages(self):
+        return ['image']
+
+    def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
+        if benchmarks is None:
+            mx.abort("Suite can only run a single benchmark per VM instance.")
+        elif len(benchmarks) != 1:
+            mx.abort("Must specify exactly one benchmark.")
+        else:
+            benchmark = benchmarks[0]
+        return self.vmArgs(bmSuiteArgs) + ["-jar",  os.path.join(self.applicationDist(), benchmark + ".jar")]
+
+    def applicationDist(self):
+        return mx.library("QUARKUS_REGISTRY_" + self.version(), True).get_path(True)
+
+    def extra_image_build_argument(self, benchmark, args):
+        return ['-J-Dsun.nio.ch.maxUpdateArraySize=100',
+                '-J-Djava.util.logging.manager=org.jboss.logmanager.LogManager',
+                '-J-DCoordinatorEnvironmentBean.transactionStatusManagerEnable=false',
+                '-J-Dio.quarkus.caffeine.graalvm.recordStats=true',
+                '-J-Dlogging.initial-configurator.min-level=500',
+                '-J-Dvertx.logger-delegate-factory-class-name=io.quarkus.vertx.core.runtime.VertxLogDelegateFactory',
+                '-J-Dvertx.disableDnsResolver=true',
+                '-J-Dio.netty.noUnsafe=true',
+                '-J-Dio.netty.leakDetection.level=DISABLED' ,
+                '-J-Dio.netty.allocator.maxOrder=3',
+                '-J-Duser.language=en',
+                '-J-Duser.country=GB',
+                '-J-Dfile.encoding=UTF-8',
+                '--features=org.hibernate.graalvm.internal.GraalVMStaticFeature,io.quarkus.hibernate.orm.runtime.graal.DisableLoggingFeature,io.quarkus.caffeine.runtime.graal.CacheConstructorsFeature,org.hibernate.graalvm.internal.QueryParsingSupport,io.quarkus.jdbc.postgresql.runtime.graal.SQLXMLFeature,io.quarkus.runner.Feature,io.quarkus.runtime.graal.ResourcesFeature,io.quarkus.runtime.graal.DisableLoggingFeature,io.quarkus.hibernate.validator.runtime.DisableLoggingFeature',
+                '-J--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED',
+                '-J--add-opens=java.base/java.text=ALL-UNNAMED',
+                '-J--add-opens=java.base/java.io=ALL-UNNAMED',
+                '-J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED',
+                '-J--add-opens=java.base/java.util=ALL-UNNAMED',
+                '-H:+AllowFoldMethods',
+                '-J-Djava.awt.headless=true',
+                '--no-fallback',
+                '-H:+ReportExceptionStackTraces',
+                '-H:-AddAllCharsets',
+                '--enable-url-protocols=http,https',
+                '-H:-UseServiceLoaderFeature',
+                '-H:+StackTrace',
+                '-J--add-exports=org.graalvm.sdk/org.graalvm.nativeimage.impl=ALL-UNNAMED',
+                '-J--add-exports=org.graalvm.nativeimage.builder/com.oracle.svm.core.jdk=ALL-UNNAMED',
+                '--exclude-config' ,
+                'io\.netty\.netty-codec',
+                '/META-INF/native-image/io\.netty/netty-codec/generated/handlers/reflect-config\.json',
+                '--exclude-config',
+                'io\.netty\.netty-handler',
+                '/META-INF/native-image/io\.netty/netty-handler/generated/handlers/reflect-config\.json'
+                ] + super(BaseQuarkusBenchmarkSuite,self).extra_image_build_argument(benchmark, args)
+
+mx_benchmark.add_bm_suite(BaseQuarkusRegistryBenchmark())
+
+class BaseMicronautMuShopBenchmark(BaseMicronautBenchmarkSuite, mx_sdk_benchmark.BaseMicroserviceBenchmarkSuite):
+    """
+    This benchmark suite is NOT yet fully functional - there is not load for measuring its runtime performance.
+    It is only useful to collect image build time metrics.
+    """
+
+    def version(self):
+        return "0.0.1"
+
+    def name(self):
+        return "mushop"
+
+    def benchmarkList(self, bmSuiteArgs):
+        return ["user", "order", "payment"]
+
+    def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
+        if benchmarks is None:
+            mx.abort("Suite can only run a single benchmark per VM instance.")
+        elif len(benchmarks) != 1:
+            mx.abort("Must specify exactly one benchmark.")
+        else:
+            benchmark = benchmarks[0]
+        return self.vmArgs(bmSuiteArgs) + ["-jar",  os.path.join(self.applicationDist(), benchmark + ".jar")]
+
+    def applicationDist(self):
+        return mx.library("MICRONAUT_MUSHOP_" + self.version(), True).get_path(True)
+
+    def extra_image_build_argument(self, benchmark, args):
+        return [
+            '--add-exports=org.graalvm.nativeimage.builder/com.oracle.svm.core.jdk=ALL-UNNAMED',
+            '--add-exports=org.graalvm.nativeimage.builder/com.oracle.svm.core.configure=ALL-UNNAMED',
+            '--add-exports=org.graalvm.sdk/org.graalvm.nativeimage.impl=ALL-UNNAMED'
+        ] + super(BaseMicronautBenchmarkSuite,self).extra_image_build_argument(benchmark, args)
+
+mx_benchmark.add_bm_suite(BaseMicronautMuShopBenchmark())
 
 class BaseShopCartBenchmarkSuite(BaseMicronautBenchmarkSuite):
     def version(self):

java-benchmarks/mx.java-benchmarks/suite.py

@@ -13,12 +13,30 @@
       "packedResource": True,
     },
 
+    "MICRONAUT_MUSHOP_0.0.1" : {
+      "urls": ["https://graalvm.oraclecorp.com/ci-legacy/slavefiles2/substratevm/native-benchmarks-experimental/mu-shop-0.0.1.zip"],
+      "digest": "sha512:41f91679c2077a9c252cfc29b8cbf7884f2e9f3c411cfe6c88d807d346178b76299bc5c3f446290b84139ca3a9f6f18c91511fc8ee90b47549b4a8c19cdb337d" ,
+      "packedResource": True
+    },
+
+    "QUARKUS_REGISTRY_0.0.1" : {
+      "urls": ["https://graalvm.oraclecorp.com/ci-legacy/slavefiles2/substratevm/native-benchmarks-experimental/quarkus-registry-0.0.1.zip"],
+      "digest": "sha512:fb798881ea406f05dd01523479b323a8e5ed6a59a545a70c06ec153ccfdb0a45be9b98d544d9fc5360007aae519a1221a7b2a794316dfc9cb9a38126ae13a9b3" ,
+      "packedResource": True
+    },
+
     "PETCLINIC_0.1.6": {
       "urls": ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/petclinic-jpa-0.1.6.zip"],
       "digest": "sha512:d16f251e4b5727c8a0e2f653a7d1d2b41967414d960e6a490d8e9fa4715f752b7f7bfc73dd069a001053bc1a4f75583abb40577c79017fef4c8f5cc972c0ed25",
       "packedResource": True,
     },
 
+    "PETCLINIC_0.1.7": {
+      "urls": ["https://graalvm.oraclecorp.com/ci-legacy/slavefiles2/substratevm/native-benchmarks-experimental/petclinic-jpa-0.1.7.zip"],
+      "digest": "sha512:fe8f34a6cbb24099e3cc68051c857e82536c6d81498c604157e6312c04e675745d6f77dd292adde006307d5319b77917bbc9e5f301501ab1e9c072870be113a5",
+      "packedResource": True,
+    },
+
     "SHOPCART_0.3.6": {
       "urls": ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/shopcart-0.3.6.zip"],
       "digest": "sha512:0ea6402e2325e8cfad2c6592c0fa36faf0c6719125f4edefc2c13740c4bfe500e8cc2726e56404c6ac6abcefb2dbf73e6d9a479d49af7e373ba665c92a4ecffc",

vm/mx.vm/mx_vm_benchmark.py

@@ -204,8 +204,17 @@ def __init__(self, vm, bm_suite, args):
                         self.stages.remove(stage)
             if self.image_vm_args is not None:
                 self.base_image_build_args += self.image_vm_args
+            self.is_runnable = self.check_runnable()
             self.base_image_build_args += self.extra_image_build_arguments
 
+        def check_runnable(self):
+            # TODO remove once there is load available for the specified benchmarks
+            if self.benchmark_suite_name in ["mushop", "quarkus"]:
+                return False
+            if self.benchmark_suite_name == "petclinic-wrk":
+                return self.bmSuite.version() == "0.1.6"
+            return True
+
     def __init__(self, name, config_name, extra_java_args=None, extra_launcher_args=None, **kwargs):
         super(NativeImageVM, self).__init__(name, config_name, extra_java_args, extra_launcher_args)
         if len(kwargs) > 0:
@@ -890,6 +899,8 @@ def run_stage_image(self, config, stages):
                 mx.run(upx_cmd, s.stdout(True), s.stderr(True))
 
     def run_stage_run(self, config, stages, out):
+        if not config.is_runnable:
+            mx.abort(f"Benchmark {config.benchmark_suite_name}:{config.benchmark_name} is not runnable.")
         image_path = os.path.join(config.output_dir, config.final_image_name)
         with stages.set_command([image_path] + config.image_run_args) as s:
             s.execute_command(vm=self)