tests/benchmark.py: tune nodes a little.

Drop the log level, don't do extra memory checks, don't dump I/O.  These are not
realistic for testing non-development nodes.

Here's the comparison, using:
	VALGRIND=0 eatmydata uv run pytest -v --benchmark-compare=0001_baseline tests/benchmark.py

Name (time in us)                                       Min                       Max                      Mean                  StdDev                    Median                     IQR            Outliers         OPS            Rounds  Iterations
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
test_invoice (NOW)                                 414.9430 (1.0)         39,115.6150 (12.35)          834.7296 (1.0)        2,274.1198 (6.59)           611.7745 (1.0)          162.0230 (1.0)          1;33  1,197.9927 (1.0)         290           1
test_invoice (0001_baselin)                        951.9740 (2.29)         3,166.4061 (1.0)          1,366.7944 (1.64)         345.1460 (1.0)          1,328.6110 (2.17)         339.3517 (2.09)        48;15    731.6389 (0.61)        221           1

test_pay (NOW)                                  36,339.2329 (87.58)       69,477.8530 (21.94)       51,719.9459 (61.96)      8,033.4262 (23.28)       52,639.5601 (86.04)      9,590.1425 (59.19)         6;0     19.3349 (0.02)         17           1
test_pay (0001_baselin)                         61,741.5591 (148.80)     108,801.6961 (34.36)       88,284.6752 (105.76)    15,875.4417 (46.00)       96,006.0760 (156.93)    27,500.9771 (169.74)        6;0     11.3270 (0.01)         13           1

test_single_payment (NOW)                       46,721.4010 (112.60)      66,027.6250 (20.85)       56,699.4597 (67.93)      5,829.7234 (16.89)       54,659.9385 (89.35)      9,810.9820 (60.55)         6;0     17.6369 (0.01)         14           1
test_single_payment (0001_baselin)              52,215.3670 (125.84)     109,608.0400 (34.62)       74,521.8032 (89.28)     16,175.6833 (46.87)       72,881.5976 (119.13)    17,668.8581 (109.05)        4;1     13.4189 (0.01)         12           1

test_forward_payment (NOW)                     108,338.2401 (261.09)     115,570.7800 (36.50)      111,353.7021 (133.40)     2,483.2338 (7.19)       111,981.6790 (183.04)     3,360.6182 (20.74)         3;0      8.9804 (0.01)         11           1
test_forward_payment (0001_baselin)            108,917.7490 (262.49)     168,348.2911 (53.17)      140,321.5990 (168.10)    22,375.2216 (64.83)      143,746.4900 (234.97)    36,363.4459 (224.43)        3;0      7.1265 (0.01)          7           1

test_start (NOW)                               299,278.4000 (721.25)     330,340.2610 (104.33)     314,121.8292 (376.32)    11,385.4700 (32.99)      314,603.4899 (514.25)    13,876.4871 (85.65)         2;0      3.1835 (0.00)          5           1
test_start (0001_baselin)                      305,928.9111 (737.28)     575,270.0820 (181.68)     419,496.8460 (502.55)   138,248.1937 (400.55)     334,207.0500 (546.29)   254,339.0035 (>1000.0)       2;0      2.3838 (0.00)          5           1

test_long_forward_payment (NOW)              1,088,077.8680 (>1000.0)  1,131,035.0260 (357.20)   1,108,896.7970 (>1000.0)   20,494.1195 (59.38)    1,098,544.8329 (>1000.0)   36,904.4899 (227.77)        3;0      0.9018 (0.00)          5           1
test_long_forward_payment (0001_baselin)     1,282,326.5721 (>1000.0)  1,450,350.8301 (458.04)   1,369,618.5776 (>1000.0)   73,432.8716 (212.76)   1,380,547.3910 (>1000.0)  132,647.3573 (818.69)        2;0      0.7301 (0.00)          5           1

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

tests/benchmark.py

@@ -12,6 +12,24 @@
 num_payments = 10000
 
 
+def get_bench_node(node_factory):
+    """Get a node which is optimized for benchmarking"""
+    # The normal log-level trace makes for a lot of IO.
+    node = node_factory.get_node(start=False, options={'log-level': 'info'})
+    # Memleak detection here creates significant overhead!
+    del node.daemon.env["LIGHTNINGD_DEV_MEMLEAK"]
+    # Don't bother recording all our io.
+    del node.daemon.opts['dev-save-plugin-io']
+    node.start()
+    return node
+
+
+def get_bench_line_graph(node_factory, num_nodes, wait_for_announce=False):
+    nodes = [get_bench_node(node_factory) for _ in range(num_nodes)]
+    node_factory.join_nodes(nodes, wait_for_announce=wait_for_announce)
+    return nodes
+
+
 @pytest.fixture
 def executor():
     ex = futures.ThreadPoolExecutor(max_workers=num_workers)
@@ -20,8 +38,8 @@ def executor():
 
 
 def test_single_hop(node_factory, executor):
-    l1 = node_factory.get_node()
-    l2 = node_factory.get_node()
+    l1 = get_bench_node(node_factory)
+    l2 = get_bench_node(node_factory)
 
     l1.rpc.connect(l2.rpc.getinfo()['id'], 'localhost:%d' % l2.port)
     l1.openchannel(l2, 4000000)
@@ -53,7 +71,7 @@ def do_pay(i, s):
 
 
 def test_single_payment(node_factory, benchmark):
-    l1, l2 = node_factory.line_graph(2)
+    l1, l2 = get_bench_line_graph(node_factory, 2)
 
     def do_pay(l1, l2):
         invoice = l2.rpc.invoice(1000, 'invoice-{}'.format(random.random()), 'desc')['bolt11']
@@ -63,7 +81,7 @@ def do_pay(l1, l2):
 
 
 def test_forward_payment(node_factory, benchmark):
-    l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+    l1, l2, l3 = get_bench_line_graph(node_factory, 3, wait_for_announce=True)
 
     def do_pay(src, dest):
         invoice = dest.rpc.invoice(1000, 'invoice-{}'.format(random.random()), 'desc')['bolt11']
@@ -73,7 +91,7 @@ def do_pay(src, dest):
 
 
 def test_long_forward_payment(node_factory, benchmark):
-    nodes = node_factory.line_graph(21, wait_for_announce=True)
+    nodes = get_bench_line_graph(node_factory, 21, wait_for_announce=True)
 
     def do_pay(src, dest):
         invoice = dest.rpc.invoice(1000, 'invoice-{}'.format(random.random()), 'desc')['bolt11']
@@ -83,7 +101,7 @@ def do_pay(src, dest):
 
 
 def test_invoice(node_factory, benchmark):
-    l1 = node_factory.get_node()
+    l1 = get_bench_node(node_factory)
 
     def bench_invoice():
         l1.rpc.invoice(1000, 'invoice-{}'.format(time()), 'desc')
@@ -92,7 +110,7 @@ def bench_invoice():
 
 
 def test_pay(node_factory, benchmark):
-    l1, l2 = node_factory.line_graph(2)
+    l1, l2 = get_bench_line_graph(node_factory, 2)
 
     invoices = []
     for _ in range(1, 100):