Updated dbconfig requests memory limit based uppon client runner checks (#188)

* Updated dbconfig requests memory limit based uppon client runner checks

* Specifying failed dbconfig memory limit

* fixes used_memory_check() arg call

* fixes used_memory_check() arg call

* Fixed resources.requests.memory for tests memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values

* Fixed resources.requests.memory for tests memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values

* Using default metrics on runner

* Fixed resources.requests.memory for tests memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values

* Fixed resources.requests.memory for tests memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values

* Fixed missing exported per test and total info on client runner

* Updated metric path of exported defaults

* Updated metric path of exported defaults

Author: filipecosta90

redis_benchmarks_specification/__cli__/args.py

@@ -30,6 +30,12 @@ def spec_cli_args(parser):
         default=SPECS_PATH_TEST_SUITES,
         help="Test suites folder, containing the different test variations",
     )
+    parser.add_argument(
+        "--defaults_filename",
+        type=str,
+        default="defaults.yml",
+        help="specify the defaults file containing spec topologies, common metric extractions,etc...",
+    )
     parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
     parser.add_argument("--branch", type=str, default="unstable")
     parser.add_argument("--commandstats-csv", type=str, default="")

redis_benchmarks_specification/__cli__/stats.py

@@ -58,7 +58,11 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
     fail_on_required_diff = args.fail_on_required_diff
     overall_result = True
     test_names = []
+    defaults_filename = args.defaults_filename
+
     for test_file in testsuite_spec_files:
+        if defaults_filename in test_file:
+            continue
         benchmark_config = {}
         requires_override = False
         test_result = True

redis_benchmarks_specification/__common__/runner.py

@@ -178,6 +178,7 @@ def exporter_datasink_common(
             ]
         },
     )
+    print(overall_end_time_metrics)
     # 7 days from now
     expire_redis_metrics_ms = 7 * 24 * 60 * 60 * 1000
     export_redis_metrics(

redis_benchmarks_specification/__runner__/args.py

@@ -26,6 +26,12 @@ def create_client_runner_args(project_name):
         default=MACHINE_NAME,
         help="Specify the running platform name. By default it will use the machine name.",
     )
+    parser.add_argument(
+        "--defaults_filename",
+        type=str,
+        default="{}/defaults.yml".format(SPECS_PATH_TEST_SUITES),
+        help="specify the defaults file containing spec topologies, common metric extractions,etc...",
+    )
     parser.add_argument("--triggering_env", type=str, default="ci")
     parser.add_argument("--setup_type", type=str, default="oss-standalone")
     parser.add_argument("--github_repo", type=str, default="redis")
@@ -178,4 +184,10 @@ def create_client_runner_args(project_name):
         type=int,
         help="override memtier test-time for each benchmark. By default will preserve test time specified in test spec",
     )
+    parser.add_argument(
+        "--override-test-runs",
+        default=1,
+        type=int,
+        help="override memtier number of runs for each benchmark. By default will run once each test",
+    )
     return parser

redis_benchmarks_specification/__runner__/runner.py

@@ -28,7 +28,10 @@
 )
 from redisbench_admin.run.metrics import extract_results_table
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
-from redisbench_admin.utils.benchmark_config import get_final_benchmark_config
+from redisbench_admin.utils.benchmark_config import (
+    get_final_benchmark_config,
+    get_defaults,
+)
 from redisbench_admin.utils.local import get_local_run_full_filename
 from redisbench_admin.utils.results import post_process_benchmark_results
 
@@ -215,6 +218,7 @@ def prepare_memtier_benchmark_parameters(
     tls_cacert=None,
     resp_version=None,
     override_memtier_test_time=0,
+    override_test_runs=1,
 ):
     benchmark_command = [
         full_benchmark_path,
@@ -253,6 +257,62 @@ def prepare_memtier_benchmark_parameters(
     if "arguments" in clientconfig:
         benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"]
     logging.info(override_memtier_test_time)
+
+    if override_test_runs > 1:
+        benchmark_command_str = re.sub(
+            "--run-count\\s\\d+",
+            "--run-count={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        benchmark_command_str = re.sub(
+            "--run-count=\\d+",
+            "--run-count={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        benchmark_command_str = re.sub(
+            '--run-count="\\d+"',
+            "--run-count={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        # short
+        benchmark_command_str = re.sub(
+            "-x\\s\\d+",
+            "-x={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        benchmark_command_str = re.sub(
+            "-x=\\d+",
+            "-x={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        benchmark_command_str = re.sub(
+            '-x="\\d+"',
+            "-x={}".format(override_test_runs),
+            benchmark_command_str,
+        )
+        if (
+            len(
+                re.findall(
+                    "--run-count={}".format(override_test_runs),
+                    benchmark_command_str,
+                )
+            )
+            == 0
+            and len(
+                re.findall(
+                    "-x={}".format(override_test_runs),
+                    benchmark_command_str,
+                )
+            )
+            == 0
+        ):
+            logging.info("adding --run-count option to benchmark run. ")
+            benchmark_command_str = (
+                benchmark_command_str
+                + " "
+                + "--run-count={}".format(override_test_runs)
+            )
+
     if override_memtier_test_time > 0:
         benchmark_command_str = re.sub(
             "--test-time\\s\\d+",
@@ -301,14 +361,25 @@ def process_self_contained_coordinator_stream(
     dry_run_count = 0
     dry_run = args.dry_run
     dry_run_include_preload = args.dry_run_include_preload
+    defaults_filename = args.defaults_filename
+    override_test_runs = args.override_test_runs
+    (
+        _,
+        default_metrics,
+        _,
+        _,
+        _,
+    ) = get_defaults(defaults_filename)
+
     for test_file in testsuite_spec_files:
+        if defaults_filename in test_file:
+            continue
         client_containers = []
 
         with open(test_file, "r") as stream:
             _, benchmark_config, test_name = get_final_benchmark_config(
                 None, stream, ""
             )
-            default_metrics = []
 
             if tls_enabled:
                 test_name = test_name + "-tls"
@@ -511,7 +582,7 @@ def process_self_contained_coordinator_stream(
                     )
 
                     used_memory_check(
-                        benchmark_required_memory, r, "start of benchmark"
+                        test_name, benchmark_required_memory, r, "start of benchmark"
                     )
 
                     logging.info("Checking if there is a keyspace check being enforced")
@@ -582,6 +653,7 @@ def process_self_contained_coordinator_stream(
                             test_tls_cacert,
                             resp_version,
                             override_memtier_test_time,
+                            override_test_runs,
                         )
 
                     client_container_image = extract_client_container_image(
@@ -652,7 +724,9 @@ def process_self_contained_coordinator_stream(
 
                     logging.info("Printing client tool stdout output")
 
-                    used_memory_check(benchmark_required_memory, r, "end of benchmark")
+                    used_memory_check(
+                        test_name, benchmark_required_memory, r, "end of benchmark"
+                    )
 
                     if args.flushall_on_every_test_end:
                         logging.info("Sending FLUSHALL to the DB")
@@ -804,14 +878,14 @@ def process_self_contained_coordinator_stream(
         )
 
 
-def used_memory_check(benchmark_required_memory, r, stage):
+def used_memory_check(test_name, benchmark_required_memory, r, stage):
     used_memory = r.info("memory")["used_memory"]
     used_memory_gb = int(math.ceil(float(used_memory) / 1024.0 / 1024.0 / 1024.0))
     logging.info("Benchmark used memory at {}: {}g".format(stage, used_memory_gb))
     if used_memory > benchmark_required_memory:
         logging.error(
-            "The benchmark specified a dbconfig resource request of memory ({}) bellow the REAL MEMORY USAGE OF: {}. FIX IT!.".format(
-                benchmark_required_memory, used_memory_gb
+            "The benchmark {} specified a dbconfig resource request of memory ({}) bellow the REAL MEMORY USAGE OF: {}. FIX IT!.".format(
+                test_name, benchmark_required_memory, used_memory_gb
             )
         )
         exit(1)

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -372,6 +372,7 @@ def process_self_contained_coordinator_stream(
     cpuset_start_pos=0,
     redis_proc_start_port=6379,
     docker_air_gap=False,
+    defaults_filename="defaults.yml",
 ):
     stream_id = "n/a"
     overall_result = False
@@ -406,6 +407,8 @@ def process_self_contained_coordinator_stream(
                 logging.info("Successfully loaded images {}".format(images_loaded))
 
             for test_file in testsuite_spec_files:
+                if defaults_filename in test_file:
+                    continue
                 redis_containers = []
                 client_containers = []
 

redis_benchmarks_specification/test-suites/defaults.yml

@@ -0,0 +1,24 @@
+
+exporter:
+  redistimeseries:
+    break_by:
+    - version
+    - commit
+    timemetric: $."ALL STATS".Runtime."Start time"
+    metrics:
+    - $."BEST RUN RESULTS".Totals."Ops/sec"
+    - $."BEST RUN RESULTS".Totals."Latency"
+    - $."BEST RUN RESULTS".Totals."Misses/sec"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."WORST RUN RESULTS".Totals."Ops/sec"
+    - $."WORST RUN RESULTS".Totals."Latency"
+    - $."WORST RUN RESULTS".Totals."Misses/sec"
+    - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Ops/sec"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Latency"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Misses/sec"
+    - $."AGGREGATED AVERAGE RESULTS (5 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Ops/sec"
+    - $."ALL STATS".Totals."Latency"
+    - $."ALL STATS".Totals."Misses/sec"
+    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"

redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-1000B-values-pipeline-10.yml

@@ -8,7 +8,7 @@ dbconfig:
     keyspacelen: 0
   resources:
     requests:
-      memory: 1g
+      memory: 52g
 tested-commands:
 - hset
 redis-topologies:
@@ -23,17 +23,7 @@ clientconfig:
     requests:
       cpus: '4'
       memory: 2g
-exporter:
-  redistimeseries:
-    break_by:
-    - version
-    - commit
-    timemetric: $."ALL STATS".Runtime."Start time"
-    metrics:
-    - $."ALL STATS".Totals."Ops/sec"
-    - $."ALL STATS".Totals."Latency"
-    - $."ALL STATS".Totals."Misses/sec"
-    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+
 tested-groups:
 - hash
 priority: 5

redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-1000B-values.yml

@@ -8,7 +8,7 @@ dbconfig:
     keyspacelen: 0
   resources:
     requests:
-      memory: 1g
+      memory: 52g
 tested-commands:
 - hset
 redis-topologies:
@@ -23,17 +23,7 @@ clientconfig:
     requests:
       cpus: '4'
       memory: 2g
-exporter:
-  redistimeseries:
-    break_by:
-    - version
-    - commit
-    timemetric: $."ALL STATS".Runtime."Start time"
-    metrics:
-    - $."ALL STATS".Totals."Ops/sec"
-    - $."ALL STATS".Totals."Latency"
-    - $."ALL STATS".Totals."Misses/sec"
-    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+
 tested-groups:
 - hash
 priority: 5

redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml

@@ -8,7 +8,7 @@ dbconfig:
     keyspacelen: 0
   resources:
     requests:
-      memory: 1g
+      memory: 9g
 tested-commands:
 - hset
 redis-topologies:
@@ -23,17 +23,7 @@ clientconfig:
     requests:
       cpus: '4'
       memory: 2g
-exporter:
-  redistimeseries:
-    break_by:
-    - version
-    - commit
-    timemetric: $."ALL STATS".Runtime."Start time"
-    metrics:
-    - $."ALL STATS".Totals."Ops/sec"
-    - $."ALL STATS".Totals."Latency"
-    - $."ALL STATS".Totals."Misses/sec"
-    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+
 tested-groups:
 - hash
 priority: 5