Merge branch 'fix-4312' into 'master'

Fixes ZSTAC-4312

Closes ZSTAC-4312

See merge request !856

Author: gitlab

cephbackupstorage/cephbackupstorage/cephagent.py

@@ -235,7 +235,7 @@ def _progress_consumer(total):
 
     @thread.AsyncThread
     def _do_import(task, fpath):
-        shell.call("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath))
+        shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath))
 
     while True:
         headers = cherrypy._cpreqbody.Part.read_headers(entity.fp)
@@ -257,7 +257,7 @@ def _do_import(task, fpath):
 
     if task.downloadedSize != task.expectedSize:
         task.fail('incomplete upload, got %d, expect %d' % (task.downloadedSize, task.expectedSize))
-        shell.call('rbd rm %s' % task.tmpPath)
+        shell.run('rbd rm %s' % task.tmpPath)
         return
 
     file_format = None
@@ -271,7 +271,7 @@ def _do_import(task, fpath):
     if file_format == 'qcow2':
         if linux.qcow2_get_backing_file('rbd:'+task.tmpPath):
             task.fail('Qcow2 image %s has backing file' % task.imageUuid)
-            shell.call('rbd rm %s' % task.tmpPath)
+            shell.run('rbd rm %s' % task.tmpPath)
             return
 
         conf_path = None
@@ -281,13 +281,13 @@ def _do_import(task, fpath):
                 conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
                 conf_path = linux.write_to_temp_file(conf)
 
-            shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' % (task.tmpPath, task.dstPath, conf_path))
-            shell.call('rbd rm %s' % task.tmpPath)
+            shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' % (task.tmpPath, task.dstPath, conf_path))
+            shell.check_run('rbd rm %s' % task.tmpPath)
         finally:
             if conf_path:
                 os.remove(conf_path)
     else:
-        shell.call('rbd mv %s %s' % (task.tmpPath, task.dstPath))
+        shell.check_run('rbd mv %s %s' % (task.tmpPath, task.dstPath))
 
     task.success()
 
@@ -544,7 +544,7 @@ def init(self, req):
             if pool.predefined and pool.name not in existing_pools:
                 raise Exception('cannot find pool[%s] in the ceph cluster, you must create it manually' % pool.name)
             elif pool.name not in existing_pools:
-                shell.call('ceph osd pool create %s 128' % pool.name)
+                shell.check_run('ceph osd pool create %s 128' % pool.name)
 
         rsp = InitRsp()
         rsp.fsid = fsid
@@ -668,7 +668,7 @@ def fail_if_has_backing_file(fpath):
 
         @rollbackable
         def _1():
-            shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
+            shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
 
         def _getRealSize(length):
             '''length looks like: 10245K'''
@@ -747,7 +747,7 @@ def _getProgress(synced):
             fail_if_has_backing_file(src_path)
             # roll back tmp ceph file after import it
             _1()
-            shell.call("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name))
+            shell.check_run("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name))
             actual_size = os.path.getsize(src_path)
         else:
             raise Exception('unknown url[%s]' % cmd.url)
@@ -766,18 +766,18 @@ def _getProgress(synced):
                     conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
                     conf_path = linux.write_to_temp_file(conf)
 
-                shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path))
-                shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
+                shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path))
+                shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
             finally:
                 if conf_path:
                     os.remove(conf_path)
         else:
-            shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
+            shell.check_run('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
         report.progress_report("100", "finish")
 
         @rollbackable
         def _2():
-            shell.call('rbd rm %s/%s' % (pool, image_name))
+            shell.check_run('rbd rm %s/%s' % (pool, image_name))
         _2()
 
 
@@ -818,8 +818,7 @@ def ping(self, req):
             rsp.failure = 'UnableToCreateFile'
             rsp.error = "%s %s" % (create_img.stderr, create_img.stdout)
         else:
-            rm_img = shell.ShellCmd('rbd rm %s' % cmd.testImagePath)
-            rm_img(False)
+            shell.run('rbd rm %s' % cmd.testImagePath)
 
         return jsonobject.dumps(rsp)
 
@@ -831,7 +830,7 @@ def delete(self, req):
         def delete_image(_):
             # in case image is deleted, we don't have to wait for timeout
             img = "%s/%s" % (pool, image_name)
-            shell.call('rbd info %s && rbd rm %s' % (img, img))
+            shell.check_run('rbd info %s && rbd rm %s' % (img, img))
             return True
 
         # 'rbd rm' might fail due to client crash. We wait for 30 seconds as suggested by 'rbd'.

kvmagent/kvmagent/plugins/ha_plugin.py

@@ -103,7 +103,7 @@ def kill_progresses_using_mount_path(mount_path):
 
     logger.warn('kill the progresses, pids:%s with mount path: %s' % (list_ps, mount_path))
     for ps_id in list_ps:
-        shell.ShellCmd("kill -9 %s || true" % ps_id)
+        linux.kill9_process(ps_id)
 
 
 def is_need_kill(vmUuid, mountPaths, isFileSystem):
@@ -219,9 +219,8 @@ def create_heartbeat_file():
             return False
 
         def delete_heartbeat_file():
-            delete = shell.ShellCmd("timeout %s rbd rm --id zstack %s -m %s" %
+            shell.run("timeout %s rbd rm --id zstack %s -m %s" %
                     (cmd.storageCheckerTimeout, cmd.heartbeatImagePath, mon_url))
-            delete(False)
 
         @thread.AsyncThread
         def heartbeat_on_ceph():
@@ -267,7 +266,7 @@ def setup_self_fencer(self, req):
         def heartbeat_file_fencer(mount_path, ps_uuid, mounted_by_zstack):
             def try_remount_fs():
                 if mount_path_is_nfs(mount_path):
-                    shell.ShellCmd("systemctl start nfs-client.target")(False)
+                    shell.run("systemctl start nfs-client.target")
 
                 while self.run_filesystem_fencer(ps_uuid, created_time):
                     if linux.is_mounted(path=mount_path) and touch_heartbeat_file():

kvmagent/kvmagent/plugins/host_plugin.py

@@ -72,7 +72,7 @@ def __init__(self):
 logger = log.get_logger(__name__)
 
 def _get_memory(word):
-    out = shell.ShellCmd("cat /proc/meminfo | grep '%s'" % word)()
+    out = shell.call("grep '%s' /proc/meminfo" % word)
     (name, capacity) = out.split(':')
     capacity = re.sub('[k|K][b|B]', '', capacity).strip()
     #capacity = capacity.rstrip('kB').rstrip('KB').rstrip('kb').strip()
@@ -177,15 +177,11 @@ def fact(self, req):
         rsp.libvirtVersion = self.libvirt_version
         rsp.ipAddresses = ipV4Addrs.splitlines()
 
-        cmd = shell.ShellCmd('cat /proc/cpuinfo | grep vmx')
-        cmd(False)
-        if cmd.return_code == 0:
+        if shell.run('grep vmx /proc/cpuinfo') == 0:
             rsp.hvmCpuFlag = 'vmx'
 
         if not rsp.hvmCpuFlag:
-            cmd = shell.ShellCmd('cat /proc/cpuinfo | grep svm')
-            cmd(False)
-            if cmd.return_code == 0:
+            if shell.run('grep svm /proc/cpuinfo') == 0:
                 rsp.hvmCpuFlag = 'svm'
 
         return jsonobject.dumps(rsp)
@@ -201,7 +197,7 @@ def capacity(self, req):
         rsp.totalMemory = _get_total_memory()
         rsp.usedMemory = used_memory
 
-        sockets = bash_o('cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l').strip('\n')
+        sockets = bash_o('grep "physical id" /proc/cpuinfo | sort -u | wc -l').strip('\n')
         rsp.cpuSockets = int(sockets)
 
         ret = jsonobject.dumps(rsp)

kvmagent/kvmagent/plugins/vm_plugin.py

@@ -3590,7 +3590,7 @@ def get_pci_info(self, req):
                 rsp.error = "%s %s" % (e, o)
                 return jsonobject.dumps(rsp)
         r_bios, o_bios, e_bios = bash.bash_roe("find /sys -iname dmar*")
-        r_kernel, o_kernel, e_kernel = bash.bash_roe("cat /proc/cmdline | grep 'intel_iommu=on'")
+        r_kernel, o_kernel, e_kernel = bash.bash_roe("grep 'intel_iommu=on' /proc/cmdline")
         if o_bios != '' and r_kernel == 0:
             rsp.hostIommuStatus = True
         else:

zstacklib/zstacklib/utils/daemon.py

@@ -3,8 +3,8 @@
 import sys, os, time, atexit
 import traceback
 from signal import SIGTERM,SIGKILL 
+from zstacklib.utils import linux
 from zstacklib.utils import log
-from zstacklib.utils import shell
 
 logger = log.get_logger(__name__)
 
@@ -61,10 +61,10 @@ def daemonize(self):
             pid = os.fork() 
             if pid > 0:
                 # exit from second parent
-                sys.exit(0) 
+                os._exit(0)
         except OSError, e: 
             sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
-            sys.exit(1) 
+            os._exit(1)
 
         # redirect standard file descriptors
         sys.stdout.flush()
@@ -79,8 +79,7 @@ def daemonize(self):
         # write pidfile
         Daemon.register_atexit_hook(self.delpid)
         atexit.register(Daemon._atexit)
-        pid = str(os.getpid())
-        file(self.pidfile,'w').write("%s\n" % pid)
+        file(self.pidfile,'w').write("%d\n" % os.getpid())
 
     def delpid(self):
         os.remove(self.pidfile)
@@ -99,15 +98,14 @@ def start(self):
             pid = None
 
         if pid:
-            pscmd = shell.ShellCmd('ps -p %s > /dev/null' % pid)
-            pscmd(is_exception=False)
-            if pscmd.return_code == 0:
+            if linux.process_exists(pid):
                 message = "Daemon already running, pid is %s\n"
                 sys.stderr.write(message % pid)
                 sys.exit(0)
 
         # Start the daemon
         self.daemonize()
+
         try:
             self.run()
         except Exception:
@@ -140,7 +138,7 @@ def stop(self):
         # Try killing the daemon process    
         start_time = time.time()
         while 1:
-            if os.path.exists('/proc/' + str(pid)):
+            if linux.process_exists(pid):
                 curr_time = time.time()
                 if (curr_time - start_time) > wait_stop:
                     os.kill(pid, SIGKILL)

zstacklib/zstacklib/utils/linux.py

@@ -3,6 +3,7 @@
 @author: frank
 '''
 import os
+import os.path
 import socket
 import subprocess
 import datetime
@@ -76,6 +77,18 @@ def rm_file_force(fpath):
     except:
         pass
 
+def process_exists(pid):
+    return os.path.exists("/proc/" + str(pid))
+
+def kill_process(pid, sig):
+    try:
+        os.kill(int(pid), int(sig))
+    except:
+        pass
+
+def kill9_process(pid):
+    kill_process(pid, 9)
+
 def cidr_to_netmask(cidr):
     cidr = int(cidr)
     return socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - cidr)) & 0xffffffff))
@@ -167,7 +180,7 @@ def get_used_disk_size(dir_path):
     return get_total_disk_size(dir_path) - get_free_disk_size(dir_path)
 
 def get_used_disk_apparent_size(dir_path):
-    output = shell.ShellCmd('du --apparent-size --max-depth=1 %s | tail -1' % dir_path)()
+    output = shell.call('du --apparent-size --max-depth=1 %s | tail -1' % dir_path)
     return long(output.split()[0])
 
 def get_disk_capacity_by_df(dir_path):
@@ -196,9 +209,7 @@ def is_mounted(path=None, url=None):
     else:
         raise Exception('path and url cannot both be None')
 
-    cmd = shell.ShellCmd(cmdstr)
-    cmd(is_exception=False)
-    return cmd.return_code == 0
+    return shell.run(cmdstr) == 0
 
 def mount(url, path, options=None):
     cmd = shell.ShellCmd("mount | grep '%s'" % path)
@@ -270,7 +281,7 @@ def umount_by_url(url):
 
 
 def get_file_size_by_http_head(url):
-    output = shell.ShellCmd('curl --head %s' % url)()
+    output = shell.call('curl --head %s' % url)
     for l in output.split('\n'):
         if 'Content-Length' in l:
             filesize = l.split(':')[1].strip()
@@ -291,7 +302,7 @@ def get_percentage(filesize, dst):
             return None
 
     def get_file_size(url):
-        output = shell.ShellCmd('curl --head %s' % url)()
+        output = shell.call('curl --head %s' % url)
         for l in output.split('\n'):
             if 'Content-Length' in l:
                 filesize = l.split(':')[1].strip()
@@ -351,7 +362,7 @@ def get_file_size(url):
         return 0
 
 def md5sum(file_path):
-    return 'md5sum is not calculated, because too time cost'
+    return 'md5sum is not calculated due to time cost'
 
     #cmd = shell.ShellCmd('md5sum %s' % file_path)
     #cmd()
@@ -550,25 +561,25 @@ def get_img_fmt(src):
 
 def qcow2_clone(src, dst):
     fmt = get_img_fmt(src)
-    shell.ShellCmd('/usr/bin/qemu-img create -F %s -b %s -f qcow2 %s' % (fmt, src, dst))()
-    shell.ShellCmd('chmod 666 %s' % dst)()
+    shell.check_run('/usr/bin/qemu-img create -F %s -b %s -f qcow2 %s' % (fmt, src, dst))
+    shell.check_run('chmod 666 %s' % dst)
 
 def raw_clone(src, dst):
-    shell.ShellCmd('/usr/bin/qemu-img create -b %s -f raw %s' % (src, dst))()
-    shell.ShellCmd('chmod 666 %s' % dst)()
+    shell.check_run('/usr/bin/qemu-img create -b %s -f raw %s' % (src, dst))
+    shell.check_run('chmod 666 %s' % dst)
 
 def qcow2_create(dst, size):
-    shell.ShellCmd('/usr/bin/qemu-img create -f qcow2 %s %s' % (dst, size))()
-    shell.ShellCmd('chmod 666 %s' % dst)()
+    shell.check_run('/usr/bin/qemu-img create -f qcow2 %s %s' % (dst, size))
+    shell.check_run('chmod 666 %s' % dst)
 
 def qcow2_create_with_backing_file(backing_file, dst):
     fmt = get_img_fmt(backing_file)
     shell.call('/usr/bin/qemu-img create -F %s -f qcow2 -b %s %s' % (fmt, backing_file, dst))
     shell.call('chmod 666 %s' % dst)
 
 def raw_create(dst, size):
-    shell.ShellCmd('/usr/bin/qemu-img create -f raw %s %s' % (dst, size))()
-    shell.ShellCmd('chmod 666 %s' % dst)()
+    shell.check_run('/usr/bin/qemu-img create -f raw %s %s' % (dst, size))
+    shell.check_run('chmod 666 %s' % dst)
 
 def create_template(src, dst):
     fmt = get_img_fmt(src)
@@ -692,10 +703,10 @@ def get_all_bridge_interface(bridge_name):
 def delete_bridge(bridge_name):
     vifs = get_all_bridge_interface(bridge_name)
     for vif in vifs:
-        shell.ShellCmd("brctl delif %s %s" % (bridge_name, vif))()
+        shell.check_run("brctl delif %s %s" % (bridge_name, vif))
 
-    shell.ShellCmd("ip link set %s down" % bridge_name)()
-    shell.ShellCmd("brctl delbr %s" % bridge_name)()
+    shell.check_run("ip link set %s down" % bridge_name)
+    shell.check_run("brctl delbr %s" % bridge_name)
 
 def find_bridge_having_physical_interface(ifname):
     output = shell.call("brctl show|sed -n '2,$p'|cut -f 1,6")
@@ -730,7 +741,7 @@ def check_ip_mask():
             return True
 
     routes = []
-    out = shell.ShellCmd('ip route')()
+    out = shell.call('ip route')
     for line in out.split('\n'):
         line.strip()
         if line:
@@ -861,17 +872,17 @@ def get_process_up_time_in_second(pid):
 
 
 def get_cpu_num():
-    out = shell.ShellCmd("cat /proc/cpuinfo | grep 'processor' | wc -l")()
+    out = shell.call("grep -c processor /proc/cpuinfo")
     return int(out)
 
 @retry(times=3, sleep_time=3)
 def get_cpu_speed():
     max_freq = '/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq'
     if os.path.exists(max_freq):
-        out = shell.ShellCmd('cat %s' % max_freq)()
+        out = file(max_freq).read()
         return int(float(out) / 1000)
 
-    cmd = shell.ShellCmd("cat /proc/cpuinfo  | grep 'cpu MHz' | tail -n 1")
+    cmd = shell.ShellCmd("grep 'cpu MHz' /proc/cpuinfo | tail -n 1")
     out = cmd(False)
     if cmd.return_code == -11:
         raise