Merge pull request kocolosk#1 from natarajaya/GPII-3624

[GPII-3624]: Improve peer discovery mechanics

Author: natarajaya

mem3_helper.py

@@ -13,22 +13,45 @@
 import backoff
 import os
 
+class PeerDiscoveryException(Exception):
+    pass
+
 def construct_service_record():
     # Drop our Pod's unique identity and replace with '_couchdb._tcp'
     return os.getenv('SRV_RECORD') or '.'.join(['_couchdb', '_tcp'] + socket.getfqdn().split('.')[1:])
 
 @backoff.on_exception(
     backoff.expo,
     dns.resolver.NXDOMAIN,
-    max_tries=10
+    max_tries=15
+)
+@backoff.on_exception(
+    backoff.expo,
+    PeerDiscoveryException,
+    max_tries=15
 )
 def discover_peers(service_record):
-    print ('Resolving SRV record', service_record)
-    answers = dns.resolver.query(service_record, 'SRV')
+    expected_peers_count = os.getenv('COUCHDB_CLUSTER_SIZE')
+    if expected_peers_count:
+        expected_peers_count = int(expected_peers_count)
+        print('Expecting', expected_peers_count, 'peers...')
+    else:
+        print('Looks like COUCHDB_CLUSTER_SIZE is not set, will not wait for DNS to fully propagate...')
+    print('Resolving SRV record:', service_record)
     # Erlang requires that we drop the trailing period from the absolute DNS
     # name to form the hostname used for the Erlang node. This feels hacky
     # but not sure of a more official answer
-    return [rdata.target.to_text()[:-1] for rdata in answers]
+    answers = dns.resolver.query(service_record, 'SRV')
+    peers = [rdata.target.to_text()[:-1] for rdata in answers]
+    peers_count = len(peers)
+    if expected_peers_count:
+        print('Discovered', peers_count, 'of', expected_peers_count, 'peers:', peers)
+        if peers_count != expected_peers_count:
+            print('Waiting for cluster DNS to fully propagate...')
+            raise PeerDiscoveryException
+    else:
+        print('Discovered', peers_count, 'peers:', peers)
+    return peers
 
 @backoff.on_exception(
     backoff.expo,
@@ -45,7 +68,7 @@ def connect_the_dots(names):
         else:
             resp = requests.put(uri, data=json.dumps(doc))
         while resp.status_code == 404:
-            print('Waiting for _nodes DB to be created ...')
+            print('Waiting for _nodes DB to be created...')
             time.sleep(5)
             resp = requests.put(uri, data=json.dumps(doc))
         print('Adding cluster member', name, resp.status_code)