Merge pull request #200 from redis/oss.cluster.client.runner

Client runner: enabled oss cluster benchmarks for string (just set and get commands) benchmarks

Author: slice4e

redis_benchmarks_specification/__cli__/stats.py

@@ -101,6 +101,11 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                 if is_memtier:
                     arguments = benchmark_config["clientconfig"]["arguments"]
                     arguments_split = arguments.split("--command")
+
+                    if len(arguments_split) == 1:
+                        # this means no arbitrary command is being used so we default to memtier default group, which is 'string'
+                        tested_groups.append("string")
+
                     for command_part in arguments_split[1:]:
                         command_part = command_part.strip()
                         command_p = command_part.split(" ", 1)[0]
@@ -248,6 +253,8 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
             for row in csv_reader:
                 if len(row) == 0:
                     continue
+                if "cmdstat_" not in row[0]:
+                    continue
                 # row variable is a list that represents a row in csv
                 cmdstat = row[0]
                 cmdstat = cmdstat.replace("cmdstat_", "")
@@ -323,8 +330,10 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                 "usecs",
                 "tracked",
                 "deprecated",
+                "usec_per_call",
                 "% count",
                 "% usecs",
+                "diff count usecs",
             ]
             import csv
 
@@ -339,10 +348,16 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     usec = row[3]
                     pct = count / total_count
                     pct_usec = "n/a"
+                    usec_per_call = "n/a"
+                    diff_pct = "n/a"
                     if usec is not None:
                         pct_usec = usec / total_usecs
+                        usec_per_call = float(usec) / float(count)
+                        diff_pct = pct_usec - pct
+                    row.append(usec_per_call)
                     row.append(pct)
                     row.append(pct_usec)
+                    row.append(diff_pct)
                     writer.writerow(row)
 
         if total_tracked_count > 0:

redis_benchmarks_specification/__common__/runner.py

@@ -178,7 +178,6 @@ def exporter_datasink_common(
             ]
         },
     )
-    print(overall_end_time_metrics)
     # 7 days from now
     expire_redis_metrics_ms = 7 * 24 * 60 * 60 * 1000
     export_redis_metrics(

redis_benchmarks_specification/__runner__/runner.py

@@ -220,6 +220,7 @@ def prepare_memtier_benchmark_parameters(
     override_memtier_test_time=0,
     override_test_runs=1,
 ):
+    arbitrary_command = False
     benchmark_command = [
         full_benchmark_path,
         "--port",
@@ -256,7 +257,9 @@ def prepare_memtier_benchmark_parameters(
     benchmark_command_str = " ".join(benchmark_command)
     if "arguments" in clientconfig:
         benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"]
-    logging.info(override_memtier_test_time)
+
+    if "--command" in benchmark_command_str:
+        arbitrary_command = True
 
     if override_test_runs > 1:
         benchmark_command_str = re.sub(
@@ -330,7 +333,7 @@ def prepare_memtier_benchmark_parameters(
             benchmark_command_str,
         )
 
-    return None, benchmark_command_str
+    return None, benchmark_command_str, arbitrary_command
 
 
 def process_self_contained_coordinator_stream(
@@ -425,13 +428,41 @@ def process_self_contained_coordinator_stream(
                         ssl_ca_certs=tls_cacert,
                         ssl_check_hostname=False,
                     )
+                    setup_name = "oss-standalone"
                     r.ping()
                     redis_conns = [r]
+                    if oss_cluster_api_enabled:
+                        redis_conns = []
+                        logging.info("updating redis connections from cluster slots")
+                        slots = r.cluster("slots")
+                        for slot in slots:
+                            # Master for slot range represented as nested networking information starts at pos 2
+                            # example: [0, 5460, [b'127.0.0.1', 30001, b'eccd21c2e7e9b7820434080d2e394cb8f2a7eff2', []]]
+                            slot_network_info = slot[2]
+                            prefered_endpoint = slot_network_info[0]
+                            prefered_port = slot_network_info[1]
+                            shard_conn = redis.StrictRedis(
+                                host=prefered_endpoint,
+                                port=prefered_port,
+                                password=password,
+                                ssl=tls_enabled,
+                                ssl_cert_reqs=ssl_cert_reqs,
+                                ssl_keyfile=tls_key,
+                                ssl_certfile=tls_cert,
+                                ssl_ca_certs=tls_cacert,
+                                ssl_check_hostname=False,
+                            )
+                            redis_conns.append(shard_conn)
+                        logging.info(
+                            "There are a total of {} shards".format(len(redis_conns))
+                        )
+                        setup_name = "oss-cluster"
+
                     redis_pids = []
-                    first_redis_pid = r.info()["process_id"]
-                    redis_pids.append(first_redis_pid)
+                    for conn in redis_conns:
+                        redis_pid = conn.info()["process_id"]
+                        redis_pids.append(redis_pid)
 
-                    setup_name = "oss-standalone"
                     github_actor = f"{tf_triggering_env}-{running_platform}"
                     dso = "redis-server"
                     profilers_artifacts_matrix = []
@@ -457,12 +488,15 @@ def process_self_contained_coordinator_stream(
                     )
                     if args.flushall_on_every_test_start:
                         logging.info("Sending FLUSHALL to the DB")
-                        r.flushall()
+                        for conn in redis_conns:
+                            conn.flushall()
 
                     benchmark_required_memory = get_benchmark_required_memory(
                         benchmark_config
                     )
-                    maxmemory = get_maxmemory(r)
+                    maxmemory = 0
+                    for conn in redis_conns:
+                        maxmemory = maxmemory + get_maxmemory(conn)
                     if benchmark_required_memory > maxmemory:
                         logging.warning(
                             "Skipping test {} given maxmemory of server is bellow the benchmark required memory: {} < {}".format(
@@ -536,7 +570,7 @@ def process_self_contained_coordinator_stream(
                         continue
 
                     if "preload_tool" in benchmark_config["dbconfig"]:
-                        data_prepopulation_step(
+                        res = data_prepopulation_step(
                             benchmark_config,
                             benchmark_tool_workdir,
                             client_cpuset_cpus,
@@ -556,12 +590,20 @@ def process_self_contained_coordinator_stream(
                             password,
                             oss_cluster_api_enabled,
                         )
+                        if res is False:
+                            logging.warning(
+                                "Skipping this test given preload result was false"
+                            )
+                            continue
                     execute_init_commands(
                         benchmark_config, r, dbconfig_keyname="dbconfig"
                     )
 
                     used_memory_check(
-                        test_name, benchmark_required_memory, r, "start of benchmark"
+                        test_name,
+                        benchmark_required_memory,
+                        redis_conns,
+                        "start of benchmark",
                     )
 
                     logging.info("Checking if there is a keyspace check being enforced")
@@ -598,6 +640,7 @@ def process_self_contained_coordinator_stream(
                             local_benchmark_output_filename
                         )
                     )
+                    arbitrary_command = False
 
                     if "memtier_benchmark" not in benchmark_tool:
                         # prepare the benchmark command
@@ -618,6 +661,7 @@ def process_self_contained_coordinator_stream(
                         (
                             _,
                             benchmark_command_str,
+                            arbitrary_command,
                         ) = prepare_memtier_benchmark_parameters(
                             benchmark_config["clientconfig"],
                             full_benchmark_path,
@@ -636,6 +680,16 @@ def process_self_contained_coordinator_stream(
                             override_test_runs,
                         )
 
+                    if (
+                        arbitrary_command
+                        and oss_cluster_api_enabled
+                        and "memtier" in benchmark_tool
+                    ):
+                        logging.warning(
+                            "Forcing skip this test given there is an arbitrary commmand and memtier usage. Check https://github.com/RedisLabs/memtier_benchmark/pull/117 ."
+                        )
+                        continue
+
                     client_container_image = extract_client_container_image(
                         benchmark_config
                     )
@@ -727,12 +781,16 @@ def process_self_contained_coordinator_stream(
                     logging.info("Printing client tool stdout output")
 
                     used_memory_check(
-                        test_name, benchmark_required_memory, r, "end of benchmark"
+                        test_name,
+                        benchmark_required_memory,
+                        redis_conns,
+                        "end of benchmark",
                     )
 
                     if args.flushall_on_every_test_end:
                         logging.info("Sending FLUSHALL to the DB")
-                        r.flushall()
+                        for r in redis_conns:
+                            r.flushall()
                     datapoint_time_ms = start_time_ms
 
                     post_process_benchmark_results(
@@ -909,8 +967,10 @@ def get_benchmark_required_memory(benchmark_config):
     return benchmark_required_memory
 
 
-def used_memory_check(test_name, benchmark_required_memory, r, stage):
-    used_memory = r.info("memory")["used_memory"]
+def used_memory_check(test_name, benchmark_required_memory, redis_conns, stage):
+    used_memory = 0
+    for conn in redis_conns:
+        used_memory = used_memory + conn.info("memory")["used_memory"]
     used_memory_gb = int(math.ceil(float(used_memory) / 1024.0 / 1024.0 / 1024.0))
     logging.info("Benchmark used memory at {}: {}g".format(stage, used_memory_gb))
     if used_memory > benchmark_required_memory:
@@ -998,6 +1058,7 @@ def data_prepopulation_step(
     password=None,
     oss_cluster_api_enabled=False,
 ):
+    result = True
     # setup the benchmark
     (
         start_time,
@@ -1019,7 +1080,11 @@ def data_prepopulation_step(
 
     if "memtier_benchmark" in preload_tool:
         override_memtier_test_time_preload = 0
-        (_, preload_command_str,) = prepare_memtier_benchmark_parameters(
+        (
+            _,
+            preload_command_str,
+            arbitrary_command,
+        ) = prepare_memtier_benchmark_parameters(
             benchmark_config["dbconfig"]["preload_tool"],
             full_benchmark_path,
             port,
@@ -1035,6 +1100,12 @@ def data_prepopulation_step(
             resp_version,
             override_memtier_test_time_preload,
         )
+        if arbitrary_command is True and oss_cluster_api_enabled:
+            logging.warning(
+                "Skipping this test given it implies arbitrary command on an cluster setup. Not supported on memtier: https://github.com/RedisLabs/memtier_benchmark/pull/117"
+            )
+            result = False
+            return result
 
         # run the benchmark
         preload_start_time = datetime.datetime.now()
@@ -1093,6 +1164,7 @@ def data_prepopulation_step(
                 client_container_stdout,
             )
         )
+    return result
 
 
 def generate_cpuset_cpus(ceil_db_cpu_limit, current_cpu_pos):

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values-pipeline-10.yml

@@ -18,7 +18,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--pipeline" "10" "--data-size" "100" --command "SET __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  arguments: '"--pipeline" "10" "--data-size" "100" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values.yml

@@ -23,7 +23,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--data-size" "100" --command "SET __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  arguments: '"--data-size" "100" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10.yml

@@ -18,7 +18,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--pipeline" "10" "--data-size" "10" --command "SET __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  arguments: '"--pipeline" "10" "--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values.yml

@@ -18,7 +18,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--data-size" "10" --command "SET __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  arguments: '"--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-1KiB-values.yml

@@ -18,7 +18,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--data-size" "1000" --command "SET __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  arguments: '"--data-size" "1000" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B-pipeline-10.yml

@@ -9,7 +9,7 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
-    arguments: '"--data-size" "100" "--command" "SET __key__ __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+    arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
   resources:
     requests:
       memory: 1g
@@ -22,10 +22,10 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--pipeline" "10" "--data-size" "100" --command "GET __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 180'
+  arguments: '"--pipeline" "10" "--data-size" "100" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180'
   resources:
     requests:
-      cpus: '2'
+      cpus: '4'
       memory: 2g
 
 tested-groups:

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B.yml

@@ -9,7 +9,7 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
-    arguments: '"--data-size" "100" "--command" "SET __key__ __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+    arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
   resources:
     requests:
       memory: 1g
@@ -22,10 +22,10 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--data-size" "100" --command "GET __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 180'
+  arguments: '--data-size 100 --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180'
   resources:
     requests:
-      cpus: '2'
+      cpus: '4'
       memory: 2g
 
 tested-groups: