Merge pull request #141 from Percona-Lab/1.0.2

1.0.2 Release

Author: timvaillancourt

Dockerfile

@@ -1,8 +1,8 @@
 FROM centos:centos7
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
-	yum install -y Percona-Server-MongoDB-32-tools zbackup && yum clean all && \
-	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.1/mongodb-consistent-backup.el7.centos.x86_64 && \
+	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all && \
+	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.2/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]

README.rst

@@ -222,6 +222,7 @@ Links
 
 - https://www.percona.com/blog/2016/07/25/mongodb-consistent-backups/
 - https://www.percona.com/blog/2017/01/09/mongodb-pit-backups-part-2/
+- https://www.percona.com/blog/2017/05/10/percona-lab-mongodb_consistent_backup-1-0-release-explained/
 - https://docs.mongodb.com/manual/reference/program/mongodump/
 - https://docs.mongodb.com/manual/reference/program/mongorestore/
 - http://zbackup.org

VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

conf/mongodb-consistent-backup.example.conf

@@ -10,9 +10,9 @@ production:
     name: default
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
-  #    binary: [path]           (default: /usr/bin/mongodump)
-  #    compression: [none|gzip] (default: true - if supported)
-  #    threads: [1-16]          (default: auto-generated - shards/cpu)
+  #    binary: [path]                (default: /usr/bin/mongodump)
+  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated - shards/cpu)
   #replication:
   #  max_lag_secs: [1+]        (default: 5)
   #  min_priority: [0-999]     (default: 0)
@@ -23,14 +23,14 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: true - if used by backup stage)
+  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
   #  resolver_threads: [1+]   (default: 2 per CPU)
   #  tailer:
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip, none if backup is compressed)
+  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -26,7 +26,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         self.replsets           = replsets
         self.sharding           = sharding
 
-        self.compression_supported = ['none', 'gzip']
+        self.compression_supported = ['auto', 'none', 'gzip']
         self.version               = 'unknown'
         self.threads_max           = 16
         self.config_replset        = False
@@ -40,11 +40,15 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         with hide('running', 'warnings'), settings(warn_only=True):
             self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True)
 
+        self.choose_compression()
+
+    def choose_compression(self):
         if self.can_gzip():
-            if self.compression() == 'none':
+            if self.compression() == 'auto':
+                logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
                 self.compression('gzip')
         elif self.compression() == 'gzip':
-            logging.warning("mongodump gzip compression requested on binary that does not support gzip!")
+            raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
     def can_gzip(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
@@ -121,8 +125,9 @@ def run(self):
                 self.authdb,
                 self.backup_dir,
                 self.binary,
+                self.version,
                 self.threads(),
-                self.do_gzip,
+                self.do_gzip(),
                 self.verbose
             )
             self.dump_threads.append(thread)
@@ -152,8 +157,9 @@ def run(self):
                     self.authdb,
                     self.backup_dir,
                     self.binary,
+                    self.version,
                     self.threads(),
-                    self.do_gzip,
+                    self.do_gzip(),
                     self.verbose
                 )]
                 self.dump_threads[0].start()

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -14,24 +14,28 @@
 
 # noinspection PyStringFormat
 class MongodumpThread(Process):
-    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary,
+    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary, version,
                  threads=0, dump_gzip=False, verbose=False):
         Process.__init__(self)
-        self.state       = state
-        self.uri         = uri
-        self.timer       = timer
-        self.user        = user
-        self.password    = password
-        self.authdb      = authdb
-        self.base_dir    = base_dir
-        self.binary      = binary
-        self.threads     = threads
-        self.dump_gzip   = dump_gzip
-        self.verbose     = verbose
-
-        self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset)
-        self.exit_code  = 1
-        self._command   = None
+        self.state     = state
+        self.uri       = uri
+        self.timer     = timer
+        self.user      = user
+        self.password  = password
+        self.authdb    = authdb
+        self.base_dir  = base_dir
+        self.binary    = binary
+        self.version   = version
+        self.threads   = threads
+        self.dump_gzip = dump_gzip
+        self.verbose   = verbose
+
+        self.timer_name        = "%s-%s" % (self.__class__.__name__, self.uri.replset)
+        self.exit_code         = 1
+        self._command          = None
+        self.do_stdin_passwd   = False
+        self.stdin_passwd_sent = False
+
         self.backup_dir = os.path.join(self.base_dir, self.uri.replset)
         self.dump_dir   = os.path.join(self.backup_dir, "dump")
         self.oplog_file = os.path.join(self.dump_dir, "oplog.bson")
@@ -60,6 +64,18 @@ def parse_mongodump_line(self, line):
         except:
             return None
 
+    def is_password_prompt(self, line):
+        if self.do_stdin_passwd and ("Enter Password:" in line or "reading password from standard input" in line):
+            return True
+        return False
+
+    def handle_password_prompt(self):
+        if self.do_stdin_passwd and not self.stdin_passwd_sent:
+            logging.debug("Received password prompt from mongodump, writing password to stdin")
+            self._process.stdin.write(self.password + "\n")
+            self._process.stdin.flush()
+            self.stdin_passwd_sent = True
+
     def wait(self):
         try:
             while self._process.stderr:
@@ -68,7 +84,11 @@ def wait(self):
                     for fd in poll[0]:
                         read = self._process.stderr.readline()
                         line = self.parse_mongodump_line(read)
-                        if line:
+                        if not line:
+                            continue
+                        elif self.is_password_prompt(read):
+                            self.handle_password_prompt()
+                        else:
                             logging.info(line)
                 if self._process.poll() != None:
                     break
@@ -85,11 +105,19 @@ def mongodump_cmd(self):
             mongodump_flags.extend(["--numParallelCollections="+str(self.threads)])
         if self.dump_gzip:
             mongodump_flags.extend(["--gzip"])
+        if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")):
+            mongodump_flags.extend(["--readPreference=secondary"])
         if self.authdb and self.authdb != "admin":
             logging.debug("Using database %s for authentication" % self.authdb)
             mongodump_flags.extend(["--authenticationDatabase", self.authdb])
         if self.user and self.password:
-            mongodump_flags.extend(["-u", self.user, "-p", self.password])
+            # >= 3.0.2 supports password input via stdin to mask from ps
+            if tuple("3.0.2".split(".")) <= tuple(self.version.split(".")):
+                mongodump_flags.extend(["-u", self.user, "-p", '""'])
+                self.do_stdin_passwd = True
+            else:
+                logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.2.0 to resolve this") 
+                mongodump_flags.extend(["-u", self.user, "-p", self.password])
         mongodump_cmd.extend(mongodump_flags)
         return mongodump_cmd
 
@@ -106,7 +134,7 @@ def run(self):
                 rmtree(self.dump_dir)
             os.makedirs(self.dump_dir)
             logging.debug("Running mongodump cmd: %s" % mongodump_cmd)
-            self._process = Popen(mongodump_cmd, stderr=PIPE)
+            self._process = Popen(mongodump_cmd, stdin=PIPE, stderr=PIPE)
             self.wait()
             self.exit_code = self._process.returncode
             if self.exit_code > 0:

mongodb_consistent_backup/Backup/Mongodump/__init__.py

@@ -5,8 +5,8 @@ def config(parser):
     parser.add_argument("--backup.mongodump.binary", dest="backup.mongodump.binary",
                         help="Path to 'mongodump' binary (default: /usr/bin/mongodump)", default='/usr/bin/mongodump')
     parser.add_argument("--backup.mongodump.compression", dest="backup.mongodump.compression",
-                        help="Compression method to use on backup (default: gzip)", default="gzip",
-                        choices=["none", "gzip"])
+                        help="Compression method to use on backup (default: auto)", default="auto",
+                        choices=["auto", "none", "gzip"])
     parser.add_argument("--backup.mongodump.threads", dest="backup.mongodump.threads",
                         help="Number of threads to use for each mongodump process. There is 1 x mongodump per shard, be careful! (default: shards/CPUs)",
                         default=0, type=int)

mongodb_consistent_backup/Backup/__init__.py

@@ -1,3 +1,8 @@
 from Backup import Backup
 
 
+def config(parser):
+    parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
+    parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
+    parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
+    return parser

mongodb_consistent_backup/Common/Config.py

@@ -51,12 +51,9 @@ def makeParser(self):
         parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose output", default=False, action="store_true")
         parser.add_argument("-H", "--host", dest="host", help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)", default="localhost", type=str)
         parser.add_argument("-P", "--port", dest="port", help="MongoDB Port (default: 27017)", default=27017, type=int)
-        parser.add_argument("-u", "--user", dest="user", help="MongoDB Authentication Username (for optional auth)", type=str)
+        parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
-        parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
-        parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
-        parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)

mongodb_consistent_backup/Common/DB.py

@@ -45,7 +45,7 @@ def connect(self):
                 conn['admin'].command({"ping":1})
         except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
             logging.error("Unable to connect to %s! Error: %s" % (self.uri, e))
-            raise OperationError(e)
+            raise DBConnectionError(e)
         if conn is not None:
             self._conn = conn
         return self._conn