Add retry decorator for flakey tests

* Update doc strings

Signed-off-by: Jhon Honce <jhonce@redhat.com>

Closes: #1302
Approved by: baude

contrib/python/podman/test/podman_testcase.py

@@ -1,3 +1,4 @@
+"""Base for podman tests."""
 import contextlib
 import functools
 import itertools
@@ -16,6 +17,7 @@ class PodmanTestCase(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        """Fixture to setup podman test case."""
         if hasattr(PodmanTestCase, 'alpine_process'):
             PodmanTestCase.tearDownClass()
 
@@ -91,6 +93,7 @@ class PodmanTestCase(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
+        """Fixture to clean up after podman unittest."""
         try:
             PodmanTestCase.alpine_process.kill()
             assert 0 == PodmanTestCase.alpine_process.wait(500)
@@ -104,5 +107,6 @@ class PodmanTestCase(unittest.TestCase):
 
     @contextlib.contextmanager
     def assertRaisesNotImplemented(self):
+        """Sugar for unimplemented varlink methods."""
         with self.assertRaisesRegex(VarlinkError, MethodNotImplemented):
             yield

contrib/python/podman/test/retry_decorator.py

@@ -0,0 +1,43 @@
+"""Decorator to retry failed method."""
+import functools
+import time
+
+
+def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, print_=None):
+    """Retry calling the decorated function using an exponential backoff.
+
+    Specialized for our unittests
+    from:
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay: initial delay between retries in seconds
+    :type delay: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    """
+    def deco_retry(f):
+        @functools.wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    if print_:
+                        print_('{}, Retrying in {} seconds...'.format(
+                            str(e), mdelay))
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry

contrib/python/podman/test/test_containers.py

@@ -2,6 +2,7 @@ import os
 import signal
 import unittest
 from test.podman_testcase import PodmanTestCase
+from test.retry_decorator import retry
 
 import podman
 
@@ -217,6 +218,8 @@ class TestContainers(PodmanTestCase):
         self.assertTrue(ctnr.running)
         self.assertTrue(ctnr.status, 'running')
 
+    # creating cgoups can be flakey
+    @retry(podman.libs.errors.ErrorOccurred, tries=16, delay=2, print_=print)
     def test_stats(self):
         self.assertTrue(self.alpine_ctnr.running)