Merge pull request #234 from timvaillancourt/flake8_opts_tweak

dbmurphy · web-flow · commit 004dfcc2b37e · 2017-10-27T13:34:12.000+01:00
Flake8 opts tweak, long-line fixes
diff --git a/Makefile b/Makefile
@@ -27,8 +27,8 @@ install: bin/$(BIN_NAME)
 	install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
 
 flake8:
-	# Ignore long-lines and space-aligned = and : for now
-	flake8 --ignore E221,E241,E501 $(PWD)/$(NAME)
+	# Ignore space-aligned = and : for now, use 160 for max-line-length
+	flake8 --count --max-line-length=160 --show-source --ignore E221,E241 $(PWD)/$(NAME)
 
 rpm: bin/$(BIN_NAME)
 	mkdir -p $(MAKE_DIR)/build/rpm/SOURCES
diff --git a/mongodb_consistent_backup/Archive/Zbackup/__init__.py b/mongodb_consistent_backup/Archive/Zbackup/__init__.py
@@ -2,9 +2,14 @@
 
 
 def config(parser):
-    parser.add_argument("--archive.zbackup.binary", dest="archive.zbackup.binary", help="Path to ZBackup binary (default: /usr/bin/zbackup)", default='/usr/bin/zbackup', type=str)
-    parser.add_argument("--archive.zbackup.cache_mb", dest="archive.zbackup.cache_mb", help="Megabytes of RAM to use as a cache for ZBackup (default: 128)", default=128, type=int)
-    parser.add_argument("--archive.zbackup.compression", dest="archive.zbackup.compression", help="Type of compression to use with ZBackup (default: lzma)", default='lzma', choices=['lzma'], type=str)
-    parser.add_argument("--archive.zbackup.password_file", dest="archive.zbackup.password_file", help="Path to ZBackup backup password file, enables AES encryption (default: none)", default=None, type=str)
-    parser.add_argument("--archive.zbackup.threads", dest="archive.zbackup.threads", help="Number of threads to use for ZBackup (default: 1-per-CPU)", default=0, type=int)
+    parser.add_argument("--archive.zbackup.binary", dest="archive.zbackup.binary", default='/usr/bin/zbackup', type=str,
+                        help="Path to ZBackup binary (default: /usr/bin/zbackup)")
+    parser.add_argument("--archive.zbackup.cache_mb", dest="archive.zbackup.cache_mb", default=128, type=int,
+                        help="Megabytes of RAM to use as a cache for ZBackup (default: 128)")
+    parser.add_argument("--archive.zbackup.compression", dest="archive.zbackup.compression", default='lzma', choices=['lzma'], type=str,
+                        help="Type of compression to use with ZBackup (default: lzma)")
+    parser.add_argument("--archive.zbackup.password_file", dest="archive.zbackup.password_file", default=None, type=str,
+                        help="Path to ZBackup backup password file, enables AES encryption (default: none)")
+    parser.add_argument("--archive.zbackup.threads", dest="archive.zbackup.threads", default=0, type=int,
+                        help="Number of threads to use for ZBackup (default: 1-per-CPU)")
     return parser
diff --git a/mongodb_consistent_backup/Archive/__init__.py b/mongodb_consistent_backup/Archive/__init__.py
@@ -2,5 +2,6 @@
 
 
 def config(parser):
-    parser.add_argument("--archive.method", dest="archive.method", help="Archiver method (default: tar)", default='tar', choices=['tar', 'zbackup', 'none'])
+    parser.add_argument("--archive.method", dest="archive.method", default='tar', choices=['tar', 'zbackup', 'none'],
+                        help="Archiver method (default: tar)")
     return parser
diff --git a/mongodb_consistent_backup/Backup/Backup.py b/mongodb_consistent_backup/Backup/Backup.py
@@ -4,6 +4,7 @@
 
 class Backup(Stage):
     def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, backup_stop=None, sharding=None):
-        super(Backup, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir, replsets=replsets, backup_stop=backup_stop, sharding=sharding)
+        super(Backup, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir,
+                                     replsets=replsets, backup_stop=backup_stop, sharding=sharding)
         self.task = self.config.backup.method
         self.init()
diff --git a/mongodb_consistent_backup/Backup/__init__.py b/mongodb_consistent_backup/Backup/__init__.py
@@ -2,7 +2,10 @@
 
 
 def config(parser):
-    parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (default: default)", default='default', type=str)
-    parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (default: /var/lib/mongodb-consistent-backup)", default='/var/lib/mongodb-consistent-backup', type=str)
-    parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
+    parser.add_argument("-n", "--backup.name", dest="backup.name", default='default', type=str,
+                        help="Name of the backup set (default: default)")
+    parser.add_argument("-l", "--backup.location", dest="backup.location", default='/var/lib/mongodb-consistent-backup', type=str,
+                        help="Base path to store the backup data (default: /var/lib/mongodb-consistent-backup)")
+    parser.add_argument("-m", "--backup.method", dest="backup.method", default='mongodump', choices=['mongodump'],
+                        help="Method to be used for backup (default: mongodump)")
     return parser
diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py
@@ -25,20 +25,24 @@ def __init__(self, option_strings, dest, nargs=0, **kwargs):
         super(PrintVersions, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, **kwargs)
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print "%s version: %s, git commit hash: %s" % (mongodb_consistent_backup.prog_name, mongodb_consistent_backup.__version__, mongodb_consistent_backup.git_commit)
+        print("%s version: %s, git commit hash: %s" % (
+            mongodb_consistent_backup.prog_name,
+            mongodb_consistent_backup.__version__,
+            mongodb_consistent_backup.git_commit
+        ))
 
         import platform
-        print "Python version: %s" % platform.python_version()
-        print "Python modules:"
+        print("Python version: %s" % platform.python_version())
+        print("Python modules:")
 
         import fabric.version
-        print "\t%s: %s" % ('Fabric', fabric.version.get_version())
+        print("\t%s: %s" % ('Fabric', fabric.version.get_version()))
 
         modules = ['pymongo', 'multiprocessing', 'yaml', 'boto', 'filechunkio']
         for module_name in modules:
             module = __import__(module_name)
             if hasattr(module, '__version__'):
-                print "\t%s: %s" % (module_name, module.__version__)
+                print("\t%s: %s" % (module_name, module.__version__))
         sys.exit(0)
 
 
@@ -61,22 +65,33 @@ def makeParser(self):
         parser = super(ConfigParser, self).makeParser()
         parser.add_argument("-V", "--version", dest="version", help="Print mongodb_consistent_backup version info and exit", action=PrintVersions)
         parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose output", default=False, action="store_true")
-        parser.add_argument("-H", "--host", dest="host", help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)", default="localhost", type=str)
+        parser.add_argument("-H", "--host", dest="host", default="localhost", type=str,
+                            help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)")
         parser.add_argument("-P", "--port", dest="port", help="MongoDB Port (default: 27017)", default=27017, type=int)
         parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
-        parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true")
-        parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true")
-        parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)", default=None, type=str)
-        parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str)
-        parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str)
+        parser.add_argument("--ssl.enabled", dest="ssl.enabled", default=False, action="store_true",
+                            help="Use SSL secured database connections to MongoDB hosts (default: false)")
+        parser.add_argument("--ssl.insecure", dest="ssl.insecure", default=False, action="store_true",
+                            help="Do not validate the SSL certificate and hostname of the server (default: false)")
+        parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", default=None, type=str,
+                            help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)")
+        parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", default=None, type=str,
+                            help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)")
+        parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", default=None, type=str,
+                            help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)")
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
-        parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
-        parser.add_argument("--rotate.max_backups", dest="rotate.max_backups", help="Maximum number of backups to keep in backup directory (default: unlimited)", default=0, type=int)
-        parser.add_argument("--rotate.max_days", dest="rotate.max_days", help="Maximum age in days for backups in backup directory (default: unlimited)", default=0, type=float)
-        parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
-        parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", help="Interval to check balancer state, in seconds (default: 3)", default=3, type=int)
+        parser.add_argument("--lock-file", dest="lock_file", default='/tmp/mongodb-consistent-backup.lock', type=str,
+                            help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)")
+        parser.add_argument("--rotate.max_backups", dest="rotate.max_backups", default=0, type=int,
+                            help="Maximum number of backups to keep in backup directory (default: unlimited)")
+        parser.add_argument("--rotate.max_days", dest="rotate.max_days", default=0, type=float,
+                            help="Maximum age in days for backups in backup directory (default: unlimited)")
+        parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", default=300, type=int,
+                            help="Maximum time to wait for balancer to stop, in seconds (default: 300)")
+        parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", default=3, type=int,
+                            help="Interval to check balancer state, in seconds (default: 3)")
         return self.makeParserLoadSubmodules(parser)
 
 
diff --git a/mongodb_consistent_backup/Oplog/Oplog.py b/mongodb_consistent_backup/Oplog/Oplog.py
@@ -100,7 +100,12 @@ def fsync(self):
 
     def autoflush(self):
         if self._oplog and self.do_flush():
-            logging.debug("Fsyncing %s (secs_since=%.2f, changes=%i, ts=%s)" % (self.oplog_file, self.secs_since_flush(), self._writes_unflushed, self.last_ts()))
+            logging.debug("Fsyncing %s (secs_since=%.2f, changes=%i, ts=%s)" % (
+                self.oplog_file,
+                self.secs_since_flush(),
+                self._writes_unflushed,
+                self.last_ts())
+            )
             return self.fsync()
 
     def close(self):
diff --git a/mongodb_consistent_backup/Oplog/Tailer/TailThread.py b/mongodb_consistent_backup/Oplog/Tailer/TailThread.py
@@ -3,7 +3,8 @@
 
 # noinspection PyPackageRequirements
 from multiprocessing import Process
-from pymongo.errors import AutoReconnect, ConnectionFailure, CursorNotFound, ExceededMaxWaiters, ExecutionTimeout, NetworkTimeout, NotMasterError, ServerSelectionTimeoutError
+from pymongo.errors import AutoReconnect, ConnectionFailure, CursorNotFound, ExceededMaxWaiters
+from pymongo.errors import ExecutionTimeout, NetworkTimeout, NotMasterError, ServerSelectionTimeoutError
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from time import sleep, time
 
diff --git a/mongodb_consistent_backup/Oplog/__init__.py b/mongodb_consistent_backup/Oplog/__init__.py
@@ -5,10 +5,16 @@
 
 
 def config(parser):
-    parser.add_argument("--oplog.compression", dest="oplog.compression", help="Compression method to use on captured oplog file (default: none)", choices=["none", "gzip"], default="none")
-    parser.add_argument("--oplog.flush.max_docs", dest="oplog.flush.max_docs", help="Maximum number of oplog document writes to trigger a flush of the backup oplog file (default: 100)", default=100, type=int)
-    parser.add_argument("--oplog.flush.max_secs", dest="oplog.flush.max_secs", help="Number of seconds to wait to flush the backup oplog file, if 'max_docs' is not reached (default: 1)", default=1, type=int)
-    parser.add_argument("--oplog.resolver.threads", dest="oplog.resolver.threads", help="Number of threads to use during resolver step (default: 1-per-CPU)", default=0, type=int)
-    parser.add_argument("--oplog.tailer.enabled", dest="oplog.tailer.enabled", help="Enable/disable capturing of cluster-consistent oplogs, required for cluster-wide PITR (default: true)", default='true', type=str)
-    parser.add_argument("--oplog.tailer.status_interval", dest="oplog.tailer.status_interval", help="Number of seconds to wait between reporting oplog tailer status (default: 30)", default=30, type=int)
+    parser.add_argument("--oplog.compression", dest="oplog.compression", choices=["none", "gzip"], default="none",
+                        help="Compression method to use on captured oplog file (default: none)")
+    parser.add_argument("--oplog.flush.max_docs", dest="oplog.flush.max_docs", default=100, type=int,
+                        help="Maximum number of oplog document writes to trigger a flush of the backup oplog file (default: 100)")
+    parser.add_argument("--oplog.flush.max_secs", dest="oplog.flush.max_secs", default=1, type=int,
+                        help="Number of seconds to wait to flush the backup oplog file, if 'max_docs' is not reached (default: 1)")
+    parser.add_argument("--oplog.resolver.threads", dest="oplog.resolver.threads", default=0, type=int,
+                        help="Number of threads to use during resolver step (default: 1-per-CPU)")
+    parser.add_argument("--oplog.tailer.enabled", dest="oplog.tailer.enabled", default='true', type=str,
+                        help="Enable/disable capturing of cluster-consistent oplogs, required for cluster-wide PITR (default: true)")
+    parser.add_argument("--oplog.tailer.status_interval", dest="oplog.tailer.status_interval", default=30, type=int,
+                        help="Number of seconds to wait between reporting oplog tailer status (default: 30)")
     return parser
diff --git a/mongodb_consistent_backup/Pipeline/Stage.py b/mongodb_consistent_backup/Pipeline/Stage.py
@@ -89,7 +89,11 @@ def run(self):
                 self.running = False
                 self.timers.stop(self.stage)
                 if self._task.completed:
-                    logging.info("Completed running stage %s with task %s in %.2f seconds" % (self.stage, self.task.capitalize(), self.timers.duration(self.stage)))
+                    logging.info("Completed running stage %s with task %s in %.2f seconds" % (
+                        self.stage,
+                        self.task.capitalize(),
+                        self.timers.duration(self.stage))
+                    )
                     self.completed = True
                 else:
                     logging.error("Stage %s did not complete!" % self.stage)
diff --git a/mongodb_consistent_backup/Replication/__init__.py b/mongodb_consistent_backup/Replication/__init__.py
@@ -3,10 +3,14 @@
 
 
 def config(parser):
-    parser.add_argument("--replication.max_lag_secs", dest="replication.max_lag_secs", help="Max lag of backup replica(s) in seconds (default: 10)", default=10, type=int)
-    parser.add_argument("--replication.min_priority", dest="replication.min_priority", help="Min priority of secondary members for backup (default: 0)", default=0, type=int)
-    parser.add_argument("--replication.max_priority", dest="replication.max_priority", help="Max priority of secondary members for backup (default: 1000)", default=1000, type=int)
-    parser.add_argument("--replication.hidden_only", dest="replication.hidden_only", help="Only use hidden secondary members for backup (default: false)", default=False, action="store_true")
+    parser.add_argument("--replication.max_lag_secs", dest="replication.max_lag_secs", default=10, type=int,
+                        help="Max lag of backup replica(s) in seconds (default: 10)")
+    parser.add_argument("--replication.min_priority", dest="replication.min_priority", default=0, type=int,
+                        help="Min priority of secondary members for backup (default: 0)")
+    parser.add_argument("--replication.max_priority", dest="replication.max_priority", default=1000, type=int,
+                        help="Max priority of secondary members for backup (default: 1000)")
+    parser.add_argument("--replication.hidden_only", dest="replication.hidden_only", default=False, action="store_true",
+                        help="Only use hidden secondary members for backup (default: false)")
     parser.add_argument("--replication.read_pref_tags", dest="replication.read_pref_tags", default=None, type=str,
                         help="Only use members that match replication tags in comma-separated key:value format (default: none)")
     return parser
diff --git a/mongodb_consistent_backup/Sharding.py b/mongodb_consistent_backup/Sharding.py
@@ -117,7 +117,8 @@ def get_balancer_state(self):
     def set_balancer(self, value):
         try:
             if self.is_gte_34():
-                # 3.4+ configsvrs dont have balancerStart/Stop, even though they're the balancer! Use self.get_mongos() to get a mongos connection for now
+                # 3.4+ configsvrs dont have balancerStart/Stop, even though they're the balancer!
+                # Use self.get_mongos() to get a mongos connection for now
                 if value is True:
                     self.get_mongos().admin_command("balancerStart")
                 else:
diff --git a/mongodb_consistent_backup/Upload/Gs/__init__.py b/mongodb_consistent_backup/Upload/Gs/__init__.py
diff --git a/mongodb_consistent_backup/Upload/S3/__init__.py b/mongodb_consistent_backup/Upload/S3/__init__.py
diff --git a/mongodb_consistent_backup/Upload/__init__.py b/mongodb_consistent_backup/Upload/__init__.py
