Ensure peer discovery node connectivity attempts for configured cluster-nodes

are carried out only once on each configured rabbit.discovery_retry_limit loop.
The erpc_call/5 abstracts a maximum 10-second timeout which can further delay
connection attempts beyond desired configured limits and intervals for nodes
which are remain unreacheable. Only single connection attempts are required at
this point, i.e. on the rabbit_peer_discovery:sync_desired_cluster/1 phase.

Author: Ayanda-D

deps/rabbit/src/rabbit_peer_discovery.erl

@@ -57,6 +57,8 @@
 -define(DEFAULT_DISCOVERY_RETRY_COUNT, 30).
 -define(DEFAULT_DISCOVERY_RETRY_INTERVAL_MS, 1000).
 
+-define(ERPC_CALL_TIMEOUT, 10_000).
+
 -define(NODENAME_PART_SEPARATOR, "@").
 
 -define(PT_PEER_DISC_BACKEND, {?MODULE, backend}).
@@ -636,7 +638,7 @@ query_node_props2([{Node, Members} | Rest], NodesAndProps, FromNode) ->
                            Node, logger, debug,
                            ["Peer discovery: temporary hidden node '~ts' "
                             "queries properties from node '~ts'",
-                            [node(), Node]], FromNode),
+                            [node(), Node]], FromNode, ?ERPC_CALL_TIMEOUT),
                          StartTime = get_node_start_time(Node, FromNode),
                          IsReady = is_node_db_ready(Node, FromNode),
                          NodeAndProps = {Node, Members, StartTime, IsReady},
@@ -726,7 +728,7 @@ is_node_db_ready(Node, FromNode) ->
     end.
 
 erpc_call(Node, Mod, Fun, Args, FromNode) ->
-    erpc_call(Node, Mod, Fun, Args, FromNode, 10000).
+    erpc_call(Node, Mod, Fun, Args, FromNode, 0).
 
 erpc_call(Node, Mod, Fun, Args, FromNode, Timeout) when Timeout >= 0 ->
     try

deps/rabbit/test/peer_discovery_classic_config_SUITE.erl

@@ -138,7 +138,8 @@ init_per_testcase(successful_discovery_with_a_subset_of_nodes_coming_online = Te
         {rmq_nodes_count, tl(NodeNames)},
         {rmq_nodes_clustered, false}
       ]),
-    NodeNamesWithHostname = [rabbit_nodes:make({Name, "localhost"}) || Name <- NodeNames],
+    NodeNamesWithHostname =
+      [rabbit_nodes:make({Name, "localhost"}) || Name <- [nonexistent1, nonexistent2 | NodeNames]],
     %% reduce retry time since we know one node on the list does
     %% not exist and not just unreachable
     %% We no longer test non-existing nodes, it just times out
@@ -150,6 +151,8 @@ init_per_testcase(successful_discovery_with_a_subset_of_nodes_coming_online = Te
           {forced_feature_flags_on_init, []},
           {cluster_nodes, {NodeNamesWithHostname, disc}},
           {cluster_formation, [
+              {discovery_retry_limit, 2},
+              {discovery_retry_interval, 100},
               {internal_lock_retries, 10}
           ]}
       ]}),