[GPII-3624]: Improve peer discovery mechanics

Author: natarajaya

mem3_helper.py

@@ -23,12 +23,29 @@ def construct_service_record():
     max_tries=10
 )
 def discover_peers(service_record):
-    print ('Resolving SRV record', service_record)
-    answers = dns.resolver.query(service_record, 'SRV')
-    # Erlang requires that we drop the trailing period from the absolute DNS
-    # name to form the hostname used for the Erlang node. This feels hacky
-    # but not sure of a more official answer
-    return [rdata.target.to_text()[:-1] for rdata in answers]
+    expected_peers_count = os.getenv('COUCHDB_CLUSTER_SIZE')
+    if expected_peers_count:
+        print('Expecting', expected_peers_count, 'peers...')
+    else:
+        print('Looks like COUCHDB_CLUSTER_SIZE is not set, will not wait for DNS...')
+    peers_count = 0
+    while str(peers_count) != expected_peers_count:
+        print('Resolving SRV record:', service_record)
+        # Erlang requires that we drop the trailing period from the absolute DNS
+        # name to form the hostname used for the Erlang node. This feels hacky
+        # but not sure of a more official answer
+        answers = dns.resolver.query(service_record, 'SRV')
+        peers = [rdata.target.to_text()[:-1] for rdata in answers]
+        peers_count = len(peers)
+        if expected_peers_count:
+            print('Discovered', peers_count, 'of', expected_peers_count, 'peers:', peers)
+            if str(peers_count) != expected_peers_count:
+                print('Waiting for cluster DNS to fully propagate...')
+                time.sleep(5)
+        else:
+            print('Discovered', peers_count, 'peers:', peers)
+            expected_peers_count = str(peers_count)
+    return peers
 
 @backoff.on_exception(
     backoff.expo,
@@ -45,7 +62,7 @@ def connect_the_dots(names):
         else:
             resp = requests.put(uri, data=json.dumps(doc))
         while resp.status_code == 404:
-            print('Waiting for _nodes DB to be created ...')
+            print('Waiting for _nodes DB to be created...')
             time.sleep(5)
             resp = requests.put(uri, data=json.dumps(doc))
         print('Adding cluster member', name, resp.status_code)