vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Restore retries of jobs not started and add some leeway to the processing time

Browse source
This commit is contained in:
Joseph Schorr 2016-08-30 13:57:26 -04:00
parent 292abb5395
commit 2fe896ba6a
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
buildman/server.py
View file

@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
WORK_CHECK_TIMEOUT = 10 WORK_CHECK_TIMEOUT = 10
TIMEOUT_PERIOD_MINUTES = 20 TIMEOUT_PERIOD_MINUTES = 20
JOB_TIMEOUT_SECONDS = 300 JOB_TIMEOUT_SECONDS = 300
SETUP_LEEWAY_SECONDS = 10
MINIMUM_JOB_EXTENSION = timedelta(minutes=2) MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
HEARTBEAT_PERIOD_SEC = 30 HEARTBEAT_PERIOD_SEC = 30
@ -164,8 +165,8 @@ class BuilderServer(object):
logger.debug('Checking for more work for %d active workers', logger.debug('Checking for more work for %d active workers',
self._lifecycle_manager.num_workers()) self._lifecycle_manager.num_workers())
job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time(), processing_time = self._lifecycle_manager.setup_time() + SETUP_LEEWAY_SECONDS
ordering_required=True) job_item = self._queue.get(processing_time=processing_time, ordering_required=True)
if job_item is None: if job_item is None:
logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT) logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
continue continue
@ -185,6 +186,7 @@ class BuilderServer(object):
except: except:
logger.exception('Exception when scheduling job: %s', build_job.repo_build.uuid) logger.exception('Exception when scheduling job: %s', build_job.repo_build.uuid)
self._current_status = BuildServerStatus.EXCEPTION self._current_status = BuildServerStatus.EXCEPTION
self._queue.incomplete(job_item, restore_retry=True, retry_after=WORK_CHECK_TIMEOUT)
return return
if schedule_success: if schedule_success: