feat: check proxmox creation status and fail if it fails

HACKING/docker-compose.yml

@@ -4,6 +4,13 @@ services:
     networks:
       - proxstar
     restart: always
+    develop:
+      watch:
+        - action: sync+restart
+          path: ../proxstar
+          target: /opt/proxstar/proxstar
+          ignore:
+            - __pycache__
 
   proxstar-postgres:
     build:
@@ -13,7 +20,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - ./proxstar-postgres/volume:/var/lib/postgresql/data:Z
+      - proxstar-postgres:/var/lib/postgresql/data:Z
     networks:
       - proxstar
     restart: always
@@ -26,6 +33,13 @@ services:
     networks:
       - proxstar
     restart: always
+    develop:
+      watch:
+        - action: sync+restart
+          path: ../proxstar
+          target: /opt/proxstar/proxstar
+          ignore:
+            - __pycache__
 
   proxstar-rq:
     build:
@@ -35,16 +49,27 @@ services:
     networks:
       - proxstar
     restart: always
+    develop:
+      watch:
+        - action: sync+restart
+          path: ../proxstar
+          target: /opt/proxstar/proxstar
+          ignore:
+            - __pycache__
+
 
   proxstar:
     build:
       context: ..
     ports:
-      - "8000:8000"
-      - "8001:8001"
+      - "8080:8080"
+      - "8081:8081"
     env_file: .env
-    entrypoint: ["gunicorn", "proxstar:app", "--bind=0.0.0.0:8000"]
+    entrypoint: ["gunicorn", "proxstar:app", "--bind=0.0.0.0:8080"]
     networks:
       - proxstar
 networks:
   proxstar:
+
+volumes:
+  proxstar-postgres:

proxstar/proxmox.py

@@ -71,3 +71,7 @@ def get_pools(proxmox, db):
             pools.append(poolid)
     pools = sorted(pools)
     return pools
+
+def get_task_status(proxmox, task_id):
+    node_name = task_id.split(':')[1]
+    return proxmox.nodes(node_name).tasks(task_id).status().get()

proxstar/starrs.py

@@ -107,6 +107,7 @@ def register_starrs(starrs, name, owner, mac, addr):
         c.execute('BEGIN')
         c.callproc('api.initialize', ('root',))
         c.callproc('api.modify_system', (name, 'comment', f'Owned by {owner}'))
+        c.callproc('api.modify_system', (name, 'type', 'Virtual Machine'))
         c.execute('COMMIT')
     except Exception as e:
         logging.exception(e)

proxstar/tasks.py

@@ -19,7 +19,7 @@
     get_template,
 )
 from proxstar.mail import send_vm_expire_email, send_rtp_vm_delete_email
-from proxstar.proxmox import connect_proxmox, get_pools
+from proxstar.proxmox import connect_proxmox, get_pools, get_task_status
 from proxstar.starrs import get_next_ip, register_starrs, delete_starrs
 from proxstar.user import User, get_vms_for_rtp
 from proxstar.vm import VM, clone_vm, create_vm
@@ -69,17 +69,26 @@ def create_vm_task(user, name, cores, memory, disk, iso):  # pylint: disable=too
             starrs = connect_starrs()
         logging.info('[{}] Creating VM.'.format(name))
         set_job_status(job, 'creating VM')
-        vmid = create_vm(proxmox, user, name, cores, memory, disk, iso)
+        vmid, task_id = create_vm(proxmox, user, name, cores, memory, disk, iso)
         logging.info('[{}] Waiting until Proxmox is done provisioning.'.format(name))
         set_job_status(job, 'waiting for Proxmox')
         timeout = 20
         retry = 0
         while retry < timeout:
+            status = get_task_status(proxmox, task_id)
+
+            if status['status'] == 'stopped' and status['exitstatus'] != 'OK':
+                logging.warning('Failed to provision, error code is {}'.format(status['exitstatus']))
+                set_job_status(job, 'Failed to provision, error code is {}'.format(status['exitstatus']))
+                delete_vm_task(vmid)
+                return
+
             if not VM(vmid).is_provisioned():
                 retry += 1
                 time.sleep(3)
                 continue
             break
+
         if retry == timeout:
             logging.info('[{}] Failed to provision, deleting.'.format(name))
             set_job_status(job, 'failed to provision')
@@ -190,17 +199,26 @@ def setup_template_task(
         get_template(db, template_id)
         logging.info('[{}] Cloning template {}.'.format(name, template_id))
         set_job_status(job, 'cloning template')
-        vmid = clone_vm(proxmox, template_id, name, user)
+        vmid, task_id = clone_vm(proxmox, template_id, name, user)
         logging.info('[{}] Waiting until Proxmox is done provisioning.'.format(name))
         set_job_status(job, 'waiting for Proxmox')
         timeout = 25
         retry = 0
         while retry < timeout:
+            status = get_task_status(proxmox, task_id)
+
+            if status['status'] == 'stopped' and status['exitstatus'] != 'OK':
+                logging.warning('Failed to provision, error code is {}'.format(status['exitstatus']))
+                set_job_status(job, 'Failed to provision, error code is {}'.format(status['exitstatus']))
+                delete_vm_task(vmid)
+                return
+
             if not VM(vmid).is_provisioned():
                 retry += 1
                 time.sleep(12)
                 continue
             break
+
         if retry == timeout:
             logging.info('[{}] Failed to provision, deleting.'.format(name))
             set_job_status(job, 'failed to provision')

proxstar/vm.py

@@ -377,7 +377,7 @@ def create_vm(proxmox, user, name, cores, memory, disk, iso):  # pylint: disable
     vmid = get_free_vmid(proxmox)
     # Make sure lingering expirations are deleted
     delete_vm_expire(db, vmid)
-    node.qemu.create(
+    task_id = node.qemu.create(
         vmid=vmid,
         name=name,
         cores=cores,
@@ -389,7 +389,7 @@ def create_vm(proxmox, user, name, cores, memory, disk, iso):  # pylint: disable
         pool=user,
         description='Managed by Proxstar',
     )
-    return vmid
+    return vmid, task_id
 
 
 # Will clone a new VM from a template, does not guarantee the
@@ -400,7 +400,8 @@ def clone_vm(proxmox, template_id, name, pool):
     # Make sure lingering expirations are deleted
     delete_vm_expire(db, vmid)
     target = get_node_least_mem(proxmox)
-    node.qemu(template_id).clone.post(
+    task_id = node.qemu(template_id).clone.post(
         newid=vmid, name=name, pool=pool, full=1, description='Managed by Proxstar', target=target
     )
-    return vmid
+
+    return vmid, task_id

requirements.txt

@@ -6,7 +6,7 @@ flask-pyoidc==3.14.3
 gunicorn==23.0.0
 markupsafe==3.0.2
 paramiko==2.11.0
-proxmoxer==2.2.0
+proxmoxer==2.3.0
 psutil==7.0.0
 psycopg2-binary==2.9.10
 python-dateutil==2.9.0