Backoff on exception instead of waiting in while loop

Author: natarajaya

mem3_helper.py

@@ -13,6 +13,9 @@
 import backoff
 import os
 
+class PeerDiscoveryException(Exception):
+    pass
+
 def construct_service_record():
     # Drop our Pod's unique identity and replace with '_couchdb._tcp'
     return os.getenv('SRV_RECORD') or '.'.join(['_couchdb', '_tcp'] + socket.getfqdn().split('.')[1:])
@@ -22,29 +25,31 @@ def construct_service_record():
     dns.resolver.NXDOMAIN,
     max_tries=10
 )
+@backoff.on_exception(
+    backoff.expo,
+    PeerDiscoveryException,
+    max_tries=10
+)
 def discover_peers(service_record):
-    expected_peers_count = os.getenv('COUCHDB_CLUSTER_SIZE')
+    expected_peers_count = int(os.getenv('COUCHDB_CLUSTER_SIZE'))
     if expected_peers_count:
         print('Expecting', expected_peers_count, 'peers...')
     else:
         print('Looks like COUCHDB_CLUSTER_SIZE is not set, will not wait for DNS...')
-    peers_count = 0
-    while str(peers_count) != expected_peers_count:
-        print('Resolving SRV record:', service_record)
-        # Erlang requires that we drop the trailing period from the absolute DNS
-        # name to form the hostname used for the Erlang node. This feels hacky
-        # but not sure of a more official answer
-        answers = dns.resolver.query(service_record, 'SRV')
-        peers = [rdata.target.to_text()[:-1] for rdata in answers]
-        peers_count = len(peers)
-        if expected_peers_count:
-            print('Discovered', peers_count, 'of', expected_peers_count, 'peers:', peers)
-            if str(peers_count) != expected_peers_count:
-                print('Waiting for cluster DNS to fully propagate...')
-                time.sleep(5)
-        else:
-            print('Discovered', peers_count, 'peers:', peers)
-            expected_peers_count = str(peers_count)
+    print('Resolving SRV record:', service_record)
+    # Erlang requires that we drop the trailing period from the absolute DNS
+    # name to form the hostname used for the Erlang node. This feels hacky
+    # but not sure of a more official answer
+    answers = dns.resolver.query(service_record, 'SRV')
+    peers = [rdata.target.to_text()[:-1] for rdata in answers]
+    peers_count = len(peers)
+    if expected_peers_count:
+        print('Discovered', peers_count, 'of', expected_peers_count, 'peers:', peers)
+        if peers_count != expected_peers_count:
+            print('Waiting for cluster DNS to fully propagate...')
+            raise PeerDiscoveryException
+    else:
+        print('Discovered', peers_count, 'peers:', peers)
     return peers
 
 @backoff.on_exception(