Merge pull request #3309 from quay/less-flaky-tests

Add automatic retry to the registry test suite to make it less flaky

test/registry/liveserverfixture.py

@@ -59,25 +59,35 @@ class liveFlaskServer(object):
       socketserver.TCPServer.server_bind = socket_bind_wrapper
       app.run(port=port, use_reloader=False)
 
-    self._process = multiprocessing.Process(target=worker, args=(self.app, 0))
-    self._process.start()
-
-    # We must wait for the server to start listening, but give up
-    # after a specified maximum timeout
-    timeout = self.app.config.get('LIVESERVER_TIMEOUT', 5)
-    start_time = time.time()
-
-    while True:
-      time.sleep(0.1)
-
-      elapsed_time = (time.time() - start_time)
-      if elapsed_time > timeout:
-        raise RuntimeError("Failed to start the server after %d seconds. " % timeout)
-
-      if self._can_connect():
-        self.app.config['SERVER_HOSTNAME'] = 'localhost:%s' % self._port_value.value
+    retry_count = self.app.config.get('LIVESERVER_RETRY_COUNT', 3)
+    started = False
+    for _ in range(0, retry_count):
+      if started:
         break
 
+      self._process = multiprocessing.Process(target=worker, args=(self.app, 0))
+      self._process.start()
+
+      # We must wait for the server to start listening, but give up
+      # after a specified maximum timeout
+      timeout = self.app.config.get('LIVESERVER_TIMEOUT', 10)
+      start_time = time.time()
+
+      while True:
+        time.sleep(0.1)
+
+        elapsed_time = (time.time() - start_time)
+        if elapsed_time > timeout:
+          break
+
+        if self._can_connect():
+          self.app.config['SERVER_HOSTNAME'] = 'localhost:%s' % self._port_value.value
+          started = True
+          break
+
+    if not started:
+      raise RuntimeError("Failed to start the server after %d retries. " % retry_count)
+
   def _can_connect(self):
     host, port = self._get_server_address()
     if port == 0: