Subprocess commands; retry on failure

lxc_net/container.py

@@ -6,7 +6,6 @@ from .jinja_template import JinjaTemplate
 
 import libvirt
 import tempfile
-import subprocess
 from pathlib import Path
 import getpass
 import weakref
@@ -75,7 +74,8 @@ class OverlayDirectory:
             self.mount_point,
         ]
 
-        subprocess.run(command)
+        util.run_cmd_retry(command)
+
         self.mounted = True
 
         self.temp_dir_cleaner = weakref.finalize(
@@ -91,7 +91,7 @@ class OverlayDirectory:
             "umount",
             self.mount_point,
         ]
-        subprocess.run(command)
+        util.run_cmd_retry(command)
         self.mounted = False
 
     def _set_perms_for_cleanup(self):
@@ -109,7 +109,8 @@ class OverlayDirectory:
             getpass.getuser(),
             self.temp_dir.resolve(),
         ]
-        subprocess.run(command_chown)
+        util.run_cmd_retry(command_chown)
+
         command_chmod = [
             "sudo",
             "chmod",
@@ -117,7 +118,7 @@ class OverlayDirectory:
             "700",
             self.temp_dir.resolve(),
         ]
-        subprocess.run(command_chmod)
+        util.run_cmd_retry(command_chmod)
 
     def cleanup_mount(self):
         self._umount()

lxc_net/util.py

@@ -3,6 +3,9 @@
 from . import settings
 import uuid
 
+import subprocess
+import sys
+
 
 class NumberedClass:
     """ A class that counts its current instance number """
@@ -125,3 +128,11 @@ class Addrv6:
         return "{base_range}:{link_id:04x}::{dev_id:04x}".format(
             base_range=settings.IPV6_RANGE, link_id=self.link_id, dev_id=self.dev_id,
         )
+
+
+def run_cmd_retry(command, *args, **kwargs):
+    rc = subprocess.run(command, *args, **kwargs)
+    while rc.returncode != 0:
+        print("Command failed. Try again:", file=sys.stderr)
+        rc = subprocess.run(command, *args, **kwargs)
+    return rc