From e8b3d1cc4a4d3de3fcdb0c291f1569b84cddfa8c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 1 Oct 2014 14:23:15 -0400 Subject: [PATCH] Phase 4 of the namespace to user migration: actually remove the column from the db and remove the dependence on serialized namespaces in the workers and queues --- ...translate_the_queue_names_to_reference_.py | 61 +++++++++++++ data/model/legacy.py | 52 +++++++---- data/queue.py | 59 +++++++------ endpoints/api/build.py | 10 ++- endpoints/api/repositorynotification.py | 16 +++- endpoints/api/user.py | 12 +++ endpoints/common.py | 6 +- endpoints/notificationhelper.py | 4 +- endpoints/registry.py | 10 ++- static/js/app.js | 1 + static/js/controllers.js | 22 ++++- static/partials/user-admin.html | 26 ++++++ test/data/test.db | Bin 630784 -> 626688 bytes test/testlogs.py | 6 +- workers/diffsworker.py | 18 ++-- workers/dockerfilebuild.py | 81 +++++++++--------- workers/notificationworker.py | 12 +-- 17 files changed, 273 insertions(+), 123 deletions(-) create mode 100644 data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py new file mode 100644 index 000000000..ae4c5d274 --- /dev/null +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -0,0 +1,61 @@ +"""Translate the queue names to reference namespace by id, remove the namespace column. + +Revision ID: 2fb36d4be80d +Revises: 3f4fe1194671 +Create Date: 2014-09-30 17:31:33.308490 + +""" + +# revision identifiers, used by Alembic. +revision = '2fb36d4be80d' +down_revision = '3f4fe1194671' + +from alembic import op +import sqlalchemy as sa + +import re +from app import app +from data.database import QueueItem, User, db + + +NAMESPACE_EXTRACTOR = re.compile(r'^([a-z]+/)([a-z0-9_]+)(/.*$)') + + +def upgrade(tables): + # Rename the namespace component of the existing queue items to reference user ids + with app.config['DB_TRANSACTION_FACTORY'](db): + for item in QueueItem.select(): + namespace_match = NAMESPACE_EXTRACTOR.match(item.queue_name) + if namespace_match is not None: + namespace_name = namespace_match.group(2) + namespace_user = User.get(User.username == namespace_name) + item.queue_name = '%s%s%s' % (namespace_match.group(1), str(namespace_user.id), + namespace_match.group(3)) + item.save() + else: + raise RuntimeError('Invalid queue name: %s' % item.queue_name) + + op.drop_index('repository_namespace_name', table_name='repository') + op.drop_column('repository', 'namespace') + + +def downgrade(tables): + # Add the namespace column back in and fill it in + op.add_column('repository', sa.Column('namespace', sa.String(length=255))) + conn = op.get_bind() + conn.execute('update repository set namespace = (select username from user where user.id = repository.namespace_user_id) where namespace is NULL') + op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True) + + # Rename the namespace component of existing queue items to reference namespace strings + with app.config['DB_TRANSACTION_FACTORY'](db): + for item in QueueItem.select(): + namespace_match = NAMESPACE_EXTRACTOR.match(item.queue_name) + if namespace_match is not None: + namespace_id = namespace_match.group(2) + namespace_user = User.get(User.id == namespace_id) + item.queue_name = '%s%s%s' % (namespace_match.group(1), + str(namespace_user.username), + namespace_match.group(3)) + item.save() + else: + raise RuntimeError('Invalid queue name: %s' % item.queue_name) diff --git a/data/model/legacy.py b/data/model/legacy.py index ba1fdd81f..effa2c8ef 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -592,6 +592,13 @@ def get_user_by_id(user_db_id): return None +def get_namespace_by_user_id(namespace_user_db_id): + try: + return User.get(User.id == namespace_user_db_id, User.robot == False).username + except User.DoesNotExist: + raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id) + + def get_user_or_org_by_customer_id(customer_id): try: return User.get(User.stripe_id == customer_id) @@ -858,6 +865,15 @@ def change_password(user, new_password): delete_notifications_by_kind(user, 'password_required') +def change_username(user, new_username): + (username_valid, username_issue) = validate_username(new_username) + if not username_valid: + raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue)) + + user.username = new_username + user.save() + + def change_invoice_email(user, invoice_email): user.invoice_email = invoice_email user.save() @@ -1676,10 +1692,21 @@ def load_token_data(code): raise InvalidTokenException('Invalid delegate token code: %s' % code) -def get_repository_build(namespace_name, repository_name, build_uuid): +def _get_build_base_query(): + return (RepositoryBuild + .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService, Repository, + Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuild) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .join(BuildTriggerService, JOIN_LEFT_OUTER) + .order_by(RepositoryBuild.started.desc())) + + +def get_repository_build(build_uuid): try: - query = list_repository_builds(namespace_name, repository_name, 1) - return query.where(RepositoryBuild.uuid == build_uuid).get() + return _get_build_base_query().where(RepositoryBuild.uuid == build_uuid).get() except RepositoryBuild.DoesNotExist: msg = 'Unable to locate a build by id: %s' % build_uuid @@ -1688,15 +1715,8 @@ def get_repository_build(namespace_name, repository_name, build_uuid): def list_repository_builds(namespace_name, repository_name, limit, include_inactive=True): - query = (RepositoryBuild - .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .switch(RepositoryBuild) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .join(BuildTriggerService, JOIN_LEFT_OUTER) + query = (_get_build_base_query() .where(Repository.name == repository_name, Namespace.username == namespace_name) - .order_by(RepositoryBuild.started.desc()) .limit(limit)) if not include_inactive: @@ -1760,21 +1780,23 @@ def create_repo_notification(repo, event_name, method_name, config): config_json=json.dumps(config)) -def get_repo_notification(namespace_name, repository_name, uuid): +def get_repo_notification(uuid): try: return (RepositoryNotification .select(RepositoryNotification, Repository, Namespace) .join(Repository) .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .where(Namespace.username == namespace_name, Repository.name == repository_name, - RepositoryNotification.uuid == uuid) + .where(RepositoryNotification.uuid == uuid) .get()) except RepositoryNotification.DoesNotExist: raise InvalidNotificationException('No repository notification found with id: %s' % uuid) def delete_repo_notification(namespace_name, repository_name, uuid): - found = get_repo_notification(namespace_name, repository_name, uuid) + found = get_repo_notification(uuid) + if (found.repository.namespace_user.username != namespace_name or + found.repository.name != repository_name): + raise InvalidNotificationException('No repository notifiation found with id: %s' % uuid) found.delete_instance() return found diff --git a/data/queue.py b/data/queue.py index 79e645ebf..73287dad6 100644 --- a/data/queue.py +++ b/data/queue.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta from data.database import QueueItem, db +from util.morecollections import AttrDict MINIMUM_EXTENSION = timedelta(seconds=20) @@ -25,17 +26,17 @@ class WorkQueue(object): def _running_jobs(self, now, name_match_query): return (QueueItem - .select(QueueItem.queue_name) - .where(QueueItem.available == False, - QueueItem.processing_expires > now, - QueueItem.queue_name ** name_match_query)) + .select(QueueItem.queue_name) + .where(QueueItem.available == False, + QueueItem.processing_expires > now, + QueueItem.queue_name ** name_match_query)) def _available_jobs(self, now, name_match_query, running_query): return (QueueItem - .select() - .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now, - ((QueueItem.available == True) | (QueueItem.processing_expires <= now)), - QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query))) + .select() + .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now, + ((QueueItem.available == True) | (QueueItem.processing_expires <= now)), + QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query))) def _name_match_query(self): return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) @@ -49,7 +50,7 @@ class WorkQueue(object): name_match_query = self._name_match_query() running_query = self._running_jobs(now, name_match_query) - running_count =running_query.distinct().count() + running_count = running_query.distinct().count() avialable_query = self._available_jobs(now, name_match_query, running_query) available_count = avialable_query.select(QueueItem.queue_name).distinct().count() @@ -89,41 +90,49 @@ class WorkQueue(object): item = None try: - item = avail.order_by(QueueItem.id).get() - item.available = False - item.processing_expires = now + timedelta(seconds=processing_time) - item.retries_remaining -= 1 - item.save() + db_item = avail.order_by(QueueItem.id).get() + db_item.available = False + db_item.processing_expires = now + timedelta(seconds=processing_time) + db_item.retries_remaining -= 1 + db_item.save() + + item = AttrDict({ + 'id': db_item.id, + 'body': db_item.body, + }) self._currently_processing = True except QueueItem.DoesNotExist: self._currently_processing = False - pass + # Return a view of the queue item rather than an active db object return item def complete(self, completed_item): with self._transaction_factory(db): - completed_item.delete_instance() + completed_item_obj = QueueItem.get(QueueItem.id == completed_item.id) + completed_item_obj.delete_instance() self._currently_processing = False def incomplete(self, incomplete_item, retry_after=300, restore_retry=False): with self._transaction_factory(db): retry_date = datetime.utcnow() + timedelta(seconds=retry_after) - incomplete_item.available_after = retry_date - incomplete_item.available = True + incomplete_item_obj = QueueItem.get(QueueItem.id == incomplete_item.id) + incomplete_item_obj.available_after = retry_date + incomplete_item_obj.available = True if restore_retry: - incomplete_item.retries_remaining += 1 + incomplete_item_obj.retries_remaining += 1 - incomplete_item.save() + incomplete_item_obj.save() self._currently_processing = False - @staticmethod - def extend_processing(queue_item, seconds_from_now): + def extend_processing(self, queue_item, seconds_from_now): new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now) # Only actually write the new expiration to the db if it moves the expiration some minimum - if new_expiration - queue_item.processing_expires > MINIMUM_EXTENSION: - queue_item.processing_expires = new_expiration - queue_item.save() + queue_item_obj = QueueItem.get(QueueItem.id == queue_item.id) + if new_expiration - queue_item_obj.processing_expires > MINIMUM_EXTENSION: + with self._transaction_factory(db): + queue_item_obj.processing_expires = new_expiration + queue_item_obj.save() diff --git a/endpoints/api/build.py b/endpoints/api/build.py index adf6f43ec..33398c15c 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -196,8 +196,9 @@ class RepositoryBuildStatus(RepositoryParamResource): @nickname('getRepoBuildStatus') def get(self, namespace, repository, build_uuid): """ Return the status for the builds specified by the build uuids. """ - build = model.get_repository_build(namespace, repository, build_uuid) - if not build: + build = model.get_repository_build(build_uuid) + if (not build or build.repository.name != repository or + build.repository.namespace_user.username != namespace): raise NotFound() can_write = ModifyRepositoryPermission(namespace, repository).can() @@ -213,7 +214,10 @@ class RepositoryBuildLogs(RepositoryParamResource): """ Return the build logs for the build specified by the build uuid. """ response_obj = {} - build = model.get_repository_build(namespace, repository, build_uuid) + build = model.get_repository_build(build_uuid) + if (not build or build.repository.name != repository or + build.repository.namespace_user.username != namespace): + raise NotFound() # If the logs have been archived, just redirect to the completed archive if build.logs_archived: diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 1fab89dd0..cc9bcb848 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -102,10 +102,14 @@ class RepositoryNotification(RepositoryParamResource): def get(self, namespace, repository, uuid): """ Get information for the specified notification. """ try: - notification = model.get_repo_notification(namespace, repository, uuid) + notification = model.get_repo_notification(uuid) except model.InvalidNotificationException: raise NotFound() + if (notification.repository.namespace_user.username != namespace or + notification.repository.name != repository): + raise NotFound() + return notification_view(notification) @require_repo_admin @@ -129,14 +133,18 @@ class TestRepositoryNotification(RepositoryParamResource): def post(self, namespace, repository, uuid): """ Queues a test notification for this repository. """ try: - notification = model.get_repo_notification(namespace, repository, uuid) + notification = model.get_repo_notification(uuid) except model.InvalidNotificationException: raise NotFound() + if (notification.repository.namespace_user.username != namespace or + notification.repository.name != repository): + raise NotFound() + event_info = NotificationEvent.get_event(notification.event.name) sample_data = event_info.get_sample_data(repository=notification.repository) notification_data = build_notification_data(notification, sample_data) - notification_queue.put([namespace, repository, notification.event.name], - json.dumps(notification_data)) + notification_queue.put([str(notification.repository.namespace_user.id), repository, + notification.event.name], json.dumps(notification_data)) return {} diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 7747addcc..b842bb466 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -139,6 +139,10 @@ class User(ApiResource): 'type': 'string', 'description': 'The user\'s email address', }, + 'username': { + 'type': 'string', + 'description': 'The user\'s username', + }, }, }, } @@ -189,6 +193,14 @@ class User(ApiResource): send_change_email(user.username, user_data['email'], code.code) else: model.update_email(user, new_email, auto_verify=not features.MAILING) + + if 'username' in user_data and user_data['username'] != user.username: + new_username = user_data['username'] + if model.get_user_or_org(new_username) is not None: + # Username already used + raise request_error(message='Username is already in use') + + model.change_username(user, new_username) except model.InvalidPasswordException, ex: raise request_error(exception=ex) diff --git a/endpoints/common.py b/endpoints/common.py index 37ae80ee8..c96a19c1d 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -213,7 +213,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, job_config = { 'docker_tags': tags, - 'repository': repo_path, + 'registry': host, 'build_subdir': subdir } @@ -221,10 +221,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, dockerfile_id, build_name, trigger, pull_robot_name=pull_robot_name) - dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({ + dockerfile_build_queue.put([str(repository.namespace_user.id), repository.name], json.dumps({ 'build_uuid': build_request.uuid, - 'namespace': repository.namespace_user.username, - 'repository': repository.name, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 6f80f83d0..cde307c30 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -30,8 +30,6 @@ def build_event_data(repo, extra_data={}, subpage=None): def build_notification_data(notification, event_data): return { 'notification_uuid': notification.uuid, - 'repository_namespace': notification.repository.namespace_user.username, - 'repository_name': notification.repository.name, 'event_data': event_data } @@ -43,5 +41,5 @@ def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[ event_name=event_name) for notification in notifications: notification_data = build_notification_data(notification, event_data) - path = [repo.namespace_user.username, repo.name, event_name] + pathargs + path = [str(repo.namespace_user.id), repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/registry.py b/endpoints/registry.py index 14bd88ce0..741601d0b 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -255,8 +255,9 @@ def put_image_layer(namespace, repository, image_id): # The layer is ready for download, send a job to the work queue to # process it. profile.debug('Adding layer to diff queue') - image_diff_queue.put([namespace, repository, image_id], json.dumps({ - 'namespace': namespace, + repo = model.get_repository(namespace, repository) + image_diff_queue.put([str(repo.namespace_user.id), repository, image_id], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, 'repository': repository, 'image_id': image_id, })) @@ -313,8 +314,9 @@ def put_image_checksum(namespace, repository, image_id): # The layer is ready for download, send a job to the work queue to # process it. profile.debug('Adding layer to diff queue') - image_diff_queue.put([namespace, repository, image_id], json.dumps({ - 'namespace': namespace, + repo = model.get_repository(namespace, repository) + image_diff_queue.put([str(repo.namespace_user.id), repository, image_id], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, 'repository': repository, 'image_id': image_id, })) diff --git a/static/js/app.js b/static/js/app.js index 27b086bab..adb11d6a9 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,6 @@ var TEAM_PATTERN = '^[a-zA-Z][a-zA-Z0-9]+$'; var ROBOT_PATTERN = '^[a-zA-Z][a-zA-Z0-9]{3,29}$'; +var USER_PATTERN = '^[a-z0-9_]{4,30}$'; $.fn.clipboardCopy = function() { if (zeroClipboardSupported) { diff --git a/static/js/controllers.js b/static/js/controllers.js index 908918631..566ce1f3a 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1676,6 +1676,8 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.logsShown = 0; $scope.invoicesShown = 0; + + $scope.USER_PATTERN = USER_PATTERN; $scope.loadAuthedApps = function() { if ($scope.authorizedApps) { return; } @@ -1752,6 +1754,24 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use }); }; + $scope.changeUsername = function() { + UserService.load(); + + $scope.updatingUser = true; + + ApiService.changeUserDetails($scope.cuser).then(function() { + $scope.updatingUser = false; + + // Reset the form. + delete $scope.cuser['username']; + + $scope.changeUsernameForm.$setPristine(); + }, function(result) { + $scope.updatingUser = false; + UIService.showFormError('#changeUsernameForm', result); + }); + }; + $scope.changeEmail = function() { UIService.hidePopover('#changeEmailForm'); @@ -1764,7 +1784,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.sentEmail = $scope.cuser.email; // Reset the form. - delete $scope.cuser['repeatEmail']; + delete $scope.cuser['email']; $scope.changeEmailForm.$setPristine(); }, function(result) { diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index a51119531..5196eb0bc 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -38,6 +38,7 @@
  • Usage Logs
    • +
  • Change Username
  • Convert to Organization
    • @@ -234,6 +235,31 @@
    + +
    +
    +
    +
    Change Username
    + +
    + +
    + + Username changed successfully + +
    +
    + + +
    +
    +
    +
    +
    +
    diff --git a/test/data/test.db b/test/data/test.db index af85d23e2016a9254ab3ad3b01876942f8d50c4f..88e3bd7a3b74e6be301c5d973a85d520468928ce 100644 GIT binary patch delta 8860 zcmeHNdwdghw$CK<$h1vTgwg`GO-rE^I>}7tl>vDq={s%HHcel&lu0rvrH}Oef>H*2 ztk>%^E5B8D!FBiUUR?oqb=y_$b#=YFy6Y;ai?~olT@~fwswiAhg@e@F^DsDso8FEJ zHvh7#-$ZnF!P`}VM~|A)S0ndC61tu-S@G103-r6(PHmO?u?TTh)}* z_^ELcSswXq_W+t+Rcwn$J=;+2*4KU*ne5q$T2AF=MK<&{poU{0 z5Si>PLuc&om>!|}0J>!V^SdL@^$nOZzdA6fnv>3&r6L=;tdaHoiYe=@Plna>8E3~A z#r#Me2%(naCMI&zK!ai0mm6O7TyxIP)d{3Ek~)}##%*5zU_=-MXzr$mwnttX96*zk z#w#Q4p>3${?c%#59}aCrb4s`RBXhSE8_*nAG8H+w9!6JPc}JvZ6rd@0 z{Ay+7Z=(ZfYH&$uq-n!8H0?OB+%>ip$CGYuYD6M~U0md~vE)QB4B@|fCA{3R6kcxM zd}eIo%oySsY2XU#3q1t|g5+~o*Dy?Jacz-QTP5<2LZ8i7C^+kiyybZnwI#)_T32~q zm6xt6s?GPh^Cgz%N{YD>8_jSOLsK+kX6%b;dNI#g?2?V81@P~ukjw_25i4v?uVm*N z$_ktg##7l)QeQ1NZ24?Koj1S2Rc5bp^Z9&bz2K>JaD`51owI_K7_Uq8)D)H1T+zNL zasmyqzCe~IUAnX&&czOM6$Hx2VO{-o=;s zJ#w{N)9nlo3w2U|n_?U2tL+TiR`jirJKK6KfpEv=tJ^GgL9{XKpx&UFrD8dxMh`!Z z9z_l+Jy|w6i?L-nN1W}wuG$g1Uv87yYusx}N(Mcp3R5^->-Af!huJ#!kh8x_unt=- zMFouH3Ho~MN@tk)2P^Y}$g(u({1P!Yl{uVsOlNT(=ZC&`c=F5E`n*<8Wm#cyYl&p_ zxa-%HcZZ!!M|<@^4d*WF?XPr7t({^1A1q-LM7v!AN1{uFYJFZ`NvMyOl+tRKv)gKM z_2+x)T^5_8XQS?^HLmUoe+^gN6V7*4)%LpF{arTjG*lhCC|wx~%iCF&AJiH<wk^ zHF^@}QcxM_ll|?=qF`T7*dOj=D~h-}##ZHlo21Y~d-F>i&hlcfSj4-_>Lo`}l_=KM zl$TX|%kuM!m_kpzjW4oQ=G&c4N2$XndM^OUbaszMSGL(;Jnd(YbiU19YOVFzi~V+s ztAt~_7>BQ>-r}xf?2gXxfQ=T``n|pOZeb{2a8!uBYZ;eozy+Q*8Z55 zG!qXPS>v!NVKAw0PY=kJ;G8Cor_8J!=iqT`hFb7;@H_ zR`(6K<$UMNp7^R+8*e5aH@;=cF?ByW&HR#atPq27Bf6_iIP|QYe-^&6OADCptf8w# zPFoNSjQl3h&S94E508WKJG5VvFDR*RaMpUAzPyq;p`e1UDlE;2@a6`T zAA_BaC(vh(`gyM9Va*f`Kbg7a%)W4QQ0`MQ7n?I{m0+&9)Xq6n$+A*6~q^im+iFd<%~LZN_2aY+KtQT`BwG|0&T#U2oX0Y(N7j-k$Tr}5Xe zY&1`gZ$xpk^8*d?f%ZLJxo#YFU@zcH@kfcd#J`Zw#)ackC3`3&u%V!TRUAB3a}ci&^tWRy6@4=}Y)VkJ z`xQ3eA)(3~#N`7A+p#Q42}!}AYzt7b5`ZeOi>CY%YokP)tOQvuNHcbStsXj;8eK)| zz;Y=?^vf`6hGGLu0Kz2M;VPn^l3=iYhGXrrpfnOtWfmg)zz)F;c$EOh|qe&9f~>nWR5;sWEO5sDZo*o;8b{= z1UG9yfmT9(S!6kd4un==&`Ji`x z!EnbEZs7M}EIBJ6`C&?iU=9e-TTT+8x&WCDiUEifV^{17Ki-Am<_Y|M?3DFqZ(@nG z0q*{3+HQ@uTYE&;gIUwe6^jSk_hAGm-izgcxBh}@UEG`ag#Yz8QJ z6&jj)H15N4U~s3Sfxsn`K;jWRW(JWXZ|Rc2hEIvO^B#GCxDPr5HFLnGeK0_HmjpUK zBN9Q{K5Qn~{u(yJ&_pCl5^`gBs^tgrBpiR|HEbplh(eeG+9t6?ux}4G7tDDdUommx zU$F%UvI^|n51|TzUmn3zz}5pWo@)+ZxkxiuegG=~NB3j%!7mPA%aGOJ@Bz#TZa4r3 z43iiMR!(A+UM7;=5~3~vA0NQ31s_d9Bp-{en>z`o!AB19gL$uGnaG-nqSrA6fxhf` z1ItEQr$RjP2DWJYBkZT!R=NxAp1J~`yS}8dCeP=nwlgJS9bL@Y>Ljkd)=5|R8YEA3 zO@*tp%Hu67cNIvU`i64aCsg|E&RTb#{_3iBm$g`UDFy-fJy9U7U2_k#WrlJudIXPK-ea94$tsQcU zGIi0Y6D$6kpb@Qq7(8>5n2&4(hffkqh^8StnMy^#nv+BtNIprVfNdv+>W%zQtj`Da4c$+0Q6ya zdE!m{Zxe^!##<2N>Rp-(9Pi@uqt3ytcNB*k=j=M@tgim{_U5TCy3MV@%p5g+Es^Yj z>FENKi-2oY~y?QOz>gU0(`XV+lb^QIHNf>BjpgD4Ys@oh45|~`0Gn>$BZ4l zAQlvCIfN&{FdsgIFGPB$qMd-zZr18Mh-7{dqME>uqfXs^7@v;xMbS-*Zm1biGbDK9 zFs?^BLHtqZ&9Wmf0J!N*;DN(e*?Zg5=ZzoXj?!!dZH%x;1 zvw?j(vFsb<&`1t=Y&((k&2q;y(R{XVzR<1NpkxQJ;2N87nQ+Rnhj$PQuOxhb2jRWO z#$Toc9ZdrMorL8|!UuK|(iO%hj_o9-uXs$qCGFe~KCwploSO-0ItOz3;df3P|Lx+q zZx@pnilg6FF2P8Dq5N_&ab8SIls&fn3`i0NR%SHv5-G*t__PZNtl>S;nBd0%Hd+gE%-6e0+C|4V`g&BuvUkp2l_ zi1ib-r-_-m0aXL6$6>AcUd(}^OUH>7fPMf;1&@D0{7R>$Zvc0m zB6#FR(EW2{HhAC^tV3r`5sX$D1v#gPdB_dGbBb85UAZ3o`Ew!z-1Rx!2wEu6sk&}( z?@^owuA}f&`|v2vAU(i*3>U%f=)iTy@C2~w7@h-<+J}$f*CHbme}qD!yASm@3iDa%TYk{1EPe4LB^@noS_`IU$a((8rsRA50{y z);F7w?}PTR-i<^ieiPQmSIjD6ii+w5XQd#x9rYYjxCp^D2v3pPsh{x@JvH@wak*Gi zTjR9Z>{W&1Pw5Ys=3T0!ovx$E%ckzp?>7Jc%d!8Pmt)|yGx{ua&bBi0F_J>iIBR=8 zU=Xs*GXWz<2-$GL0350CM=d9S}99kTCx|;)DExCeN_RKBMt7a*3T@Md- zYsnU3daIJHrrRS2;Xsd$4B_UVrXFb&BO7};a1Bak;o_X1c6x-!8}PXcC6kGT&o*sV zQ}_to$Af23xJ23-!a2l6{?x~TMvPp6CWHBJgZnXZn<)id_!l+ZcHzrPD;UMeAfAX;Z{L>lD5p9!)Vop-zH1y(&d zK+M{8`>m>(ij)uW;4M9jBy-Pmzf#k4BOk+o>q#<%&fDQ_0w0oO1HN!p#xc*r$VbB* z*c3-5qxh+3S@5Gc65w}jk`*2 zYV^g(t%gq1d$+WBs$=9a`mj-5>|+%8upW||9{ z-cD+Hn;=ee?4vKLDUKMqP0;m}xliN($13Q$;l+|8s(E>g+*Od=?EPye z!K!9Rt}W@e<>RZ#rHM!ySk^^;WSE{iJlivL4jK421-YQ5n@lm$*KN0`DT5aGCkPh6TZ&%Izka-Tg*h8)%a{hb9&(!oj$UFzSd&w4@ zntf(LV=rW$1=IS-c{soO*V8>cka-q_`^Xd`0ozEcDcz8H4*af?a+l_#V0i+}{tGH-EAJHZ{HTYv!YVj|{*?u+6{T(bxf*XTj`2a-KoE%0AQ6el9-v zw+OjlXb>)@{p8dkHKh$Q&w=L$$<=7a))y3D9g4!|c;^+>+zOfJz?(xbk_ErJ0dDIx zka-Sl9)@@_&wX44J{*S3&z`aI_nr`Bo(1X%nSw5Q{jeSUXaq99BzN;&L%|sH>mc*l z|M@qkY7Sh?{5r_|lKZ{{YPvth{Cdbd)AB-2qa0&?6f(c`2FDGa)iLHrA@d2>4#A{t zjxj$9na?VEW;%FJ!j%|5gvp54S zP$&=(C5-Mzn;F& zvz&9!a?kIabMMS}%g!ZRb}!kwaMo&_PWL1DH}$*mjvNjWsdv&bNWx9;gcw_X(LG=? zcJ;%{wfgTb95~+ zxOLDHT-$Rky6CXuFTv+~2GPjCGhYUk-f?vPbtD};(z^rAO1k;x;O0ITy3qLu8a&ch ziO%`(wKGAgAE5JBxg&$W?;kWRfAIFbnmO*gS}M4?+Y;P3pqh-96JuKP-1GG8upfzo zepLA4?k|Hk4!WY}JYBcWK6}c~bzc*fVB%05YJBqDmxJOEG_l~3Rl(PW22p8Ge_616 zcpQyv9(X5sVt5C-Vz>3<;MNfrntso(nu8}sD$(rCpX?3hjRG|Hou;_pq0vFpN^p3M zW+tZ{3(fbo4Js<_IFb-NxnT#o{6F9J1n=GGLYKC`HxWF!u@X(MTUi<`*#yw22k!n! z@ZC*=XvB@_w&1m!$5DP3QFiCp4m?3d?r%bZL*0Dvz*v0jJtHvu+t$Ft9V_5r>-A^H zCeMx`_EA@3MOjs`)n+SF3awT7Y(q)CWaR~xsVHAvXDyH#t2w!$v9zwSq@uEfV=HBN zl(=d!pDW=V`OZ2AOEWx0D-_F{S!S6aE@RnTL6B*I0grx#q`E|>gLODu4b@Chbv`RW zO$~KoQJswur9xYgy@W4Q_=;+_geeoMtLp7yQGK<|Nf+{Ltx%TFlt@?9&oi>Zg0xde zUfiXoS%K#`FnkIzkJYvnws%RQqBh9h_4PGv_7P9RK(58wZmXyqp;?R4&Pe@%Kt-!p zE^ta-m()_$UL5FRSM|5L1MPje-mcEet<$-JB8UPvL`3W6YgiVkBjC^4tGaz^ZyM`P zW7xEUhW3)KK)uCV(Ve@Vu4(a?RMqfZ1EY0Pqfc%r5}me+&RmDwQ`0k0+EZIuE>y6i zRRw`A=3iK5Wkp~(5cnD~Uz<<6tNRN4?7&c4QGr}oTTsJ|Sk%F;!C_baXs+gj^-0$u#SFd@nUqbQ*DG?MP3>k4d2AzNHr zLDLoGPNurXDGB+4v&z1@th~IuuG~>YJ8N9|C5=*DeNk~id84DGp&`Fav^$*@&YJwo z@PShhWm;ru@WE-sER2dn6|9x5V}0@nCrXlq8E~m~S9Pvu&{Im zM|H#chHAQ2_V^tA^|pSuMaeI(7+PIcSPx!+qGL1D6>6m=fm4(reYB3!Y0-p^AYm@~ z)WLqYr$f#5_4jspy86o-EA3^qm5OZTX|bT*;ap9Zm)WbC0+1SM$7;4j>*M=FcHgr%Ka$cF< z@C|Xx`p%g>d8Sz(X*TwlvP{p!y#32r=GS7ziZM8E#Bj9>n~GV=zibB+TlJ@^GE0OChs|DH#W6O8ZE!hwd$rKuDzOPw4CAaRaJZ`K z3g9YV1D-#wPX{H(_4Z8J&Qyps^lH{sQd8%suPJNf*-C+Pa`}y{Q!3$VYXs-&Mr-LL z6#2+fEeUSU?_(s2_DTxHdjyG+8C9VCqQEO~U8g11->Y_aEpz)i+XDS+Uw?mByBe78 zR)U$Pm+{;(I<%o+8IhidH5@UeeHXRc%m$i_xzBWu?t5^z10FCNPJ`|&0}94d4ZaJv zH^SS=@l-UiDUEllenyo!O7@8!ikE#H<#E#Zgp;9HG4V+;9EL`B3`kq<`YL_HHd7JWL#Yr4U-4({o)`TiK#`ZLr6dS68rOn&$?^xarKyui)y zAv7O-qQOdDdAjFZ+>QRyRQX*X&}SH&*f1kho)P^t12e$KdMRFHMasi^Rf-cNkJl%_ZG_jY zH&~E!0*DR@sBV=J1cvha1)k#F(1Ogu9kEySdp$-)_IVV)Ghvq1jbBh!# zJr&lW+egW~pP?mQP~5z?kqikhL<(jIXLVv}A`G7_czGY?7Q9f{D~Yf|+ziEueyGV$ z^SsZ$jtF_42rW-}4n{k%9GIAr>`|et#4-W}_by(_*y2fTyF)3x)z1Hzs&} zeo^*wq0rLMXXt_AW}xGu;FknOR8_&7hlYGknSoRbjJ7+_Kj04NSw)t;tm>w`Fuf?A z^>`>ZF9?*I@v%JPX8m3_-);ygSctSwPZ~aIba$x3dJx}*S;iRA$0)F{DL;IUqIjO= zDMjJ^6wPvsA~B*DMz#_rZ76Wew2r>EkPdzAxsy56S^L`hyLv}*d4cyp9AEU3w35xq`-lOYp5j&#*ti#40Cw!f6mWbmmJ%IWiURIVH8o31Tnz^HViK|jXsJXp_~Txz1eww`33X`* z(g5z*kEMXWzkw|T_r8gxg|3X?w{K!eQT?jBv$@R&^~5xWM{Rx+j-vNt>0sh(|#yx+K)wpvpEC~qW0nuVAlaG28`{;W=7&$*wP78gto|SjIUG> zEf4HG3_V+Z6eG7C#pZ#hLSpbvBs|9Waya0c@Bqk6LRvw}UMvCp<1o~E=qSX-3rxqb z)abT?8k@DwX0Eg5SJ=SrY+^1L`8{Hu>^+91Bhcr2j$;|f`rzA!IPk`CEL*=(1xfpG z8oaa*Uj)|g!x_X6?%#(mMLK|eAHER0vky;uWH%POZMPAfMBl=8j<0c)RoQH-xr$=W z&dFjiBjj`WWrC|%ax%2DkhKa@u~^PJM6SNHve=dHvK3d_Yila3N|{(y#g^3&moLU_ zF0aS}&BFIW_zR|0S8HKnEB+%(8Y9a-)knK2nuG0(=NLbw$Zpus1;5W9dJW(al*#9w z$BSY!W037SWII}cp2Qx;9r!24$0B+n-iz#uBBDTaV)SD%6Q*R-ZMqg%4jG2;=d=Lw z4O|9i_TvP2dJi@W+JYY*XN zNZ$<2uN=aQh)!dCb~bXe6~w-Q&js{h+=TRpv$x?ee!U*La^xs(0XH4PlflwscskMr z9LL}U=O2fJj$<=?Gl7?n;c-YeIDQOY1|ARbPaVTep!hhRgY<-?7J;qD;o;#-Snu#E z8r*ykj{|={j?Xb@hF;K}NeF<-g3KRg5)t5)Od|i@vzt4 zwBg(fiH@!oHPGKXGIicCIi5vG`iZx2vg+c^C)~2bHD6XhQm%3TWh|H#5(6SES7M2@ zB2Rv9F%Gm}EXL(Zw5ZSw*m9roPWyMstJF<(kr8lHr$cmBT?-j;g$?MmQuqDFf^V>Q zY`ZZN(A$kD`1Ae7v~L&$PmWa~8@*pm)Z`KEJa1!9*FnW$E+8NtuYL@jcA$czQ-GZS;puTH5@ zuqB09h~VJ*6rvKj4ZM~@wCGy~z=3Qc1FX*>5 z{2IOlxh||j$|aT{8^G8eJRZ4z670cgMBlIxJZd4#fXgJ}z)=gac)^-YP*c2qtS#VE zhub>cEoyTg+~L79_+18(1WGcAbiKM6{4|3|M{WSGWe|A=EqM$?WWadd2zq{kEC8+y z!j4`iT^_8HhC%I;m}a3GrhO+k=Al}Wc4a_UT>*VPVo1?2usoG01bb2l0^F2JI49mR zZc94zaX7Vp*8!uvCR>SyA_T^QK zUBVu_#5j5p|FU{Ed^`K1=kCk9LFg-;-t;*<>VWB#>GPOPF~6Eayl=vT4Ulx_A25+%@@hGZE>SUoMM6O=Dp}L5;%6rA&S{g6j~0BK7Cq zX2!bs0#|8CZK=&ENK!dlKJjznuqpjgA^jW!MP4>^H}R_Z|GyFb-+Utk2mV2%p|SIP zof99y{)R5@U-LO&J|-$nN^n=KmT=9Cudx((_G6+M=TB7XhL;6j=;1-_C&Vga&LZwE z&8!5M_X^Z@8&zeA!6x`CsgV~=FX}IO++Ge{LJO=N( zKPBRgS&x4aqa_GIx?cb!IAi+hU3M+O5>$o+ z@W+!xGfI5&Kpe1qK@1wBf@>!=GZm~F7QowIz?mppGoRCvbAlg3!uP%;{Ak*2xQLwi zl5pY4f?vY>TTa6` zrQdoA(GnI1%Qp$&*lFmx@!_;gU_Ap}zeXCdY37vRy3GRk=NX~}O+C0_BUpQu*nwt> zcRlnJNm)_kiBDDn#!6P27Ctg~Kuc)67+WjZjOz~k`p|Gg7~6bul`-$LeaV`+K8$TX z*K0 zt-Wd*+l~;n#blf@FS+QqT0&(Q+hVdA%r=&O}}~^nsRK z7RI)e^rP9CNd!1iO1kirvAN^+k}$SqWISrB-_{AXmyrNpw(F+F!$o0i%gI$HJn^%q zG_!36wiNhFInX&$rWCEei++I7^kJ{?p&iK zSi{&>Le~>t2Pc8G3c8+9mHMb=&I@B(MYf>guazjU)=us~xeg1gC> z8@-1$^8iGg2d{X@R-th9 z@{tL~6<=p>(-L|h;yn0`k6ec@$V!_(+zk=uz#5gbqA5FY7ucyn#50R6gIaRn0^%Wm zNBz)9>e^#3Hg!V8Ik2FGOpl&DnDHBX$JD&wzhdNo;TC9Yo;nHBs~sZFga2wF*P*j_ zJ$VbTw8F4qy2l)vxeX%DgST4YOvoKMf6$U!A>ur^z758a<-az86KxRj%xe4Ld4^TpTM<*<}l)&5b-%L-~Cfy4M5lP9O4t2`Pwkz0qE(9 j2XDO#tnGq`FWdg~;6yj+N=CV?f4~aA9nw~QN+$ge8%4cb diff --git a/test/testlogs.py b/test/testlogs.py index cd4ea6c9d..a6574e331 100644 --- a/test/testlogs.py +++ b/test/testlogs.py @@ -67,8 +67,7 @@ class TestBuildLogs(RedisBuildLogs): (phase, status) = status_wrapper from data import model - build_obj = model.get_repository_build(self.namespace, self.repository, - self.test_build_id) + build_obj = model.get_repository_build(self.test_build_id) build_obj.phase = phase build_obj.save() @@ -88,8 +87,7 @@ class TestBuildLogs(RedisBuildLogs): total_commands = random.randint(5, 20) for command_num in range(1, total_commands + 1): command_weight = random.randint(50, 100) - script.append(self._generate_command(command_num, total_commands, - command_weight)) + script.append(self._generate_command(command_num, total_commands, command_weight)) # we want 0 logs some percent of the time num_logs = max(0, random.randint(-50, 400)) diff --git a/workers/diffsworker.py b/workers/diffsworker.py index 563c61352..3a6a8459a 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -1,30 +1,28 @@ import logging -import argparse from app import image_diff_queue -from data.model import DataModelException +from data import model from endpoints.registry import process_image_changes from workers.worker import Worker -root_logger = logging.getLogger('') -root_logger.setLevel(logging.DEBUG) - -FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' -formatter = logging.Formatter(FORMAT) - logger = logging.getLogger(__name__) class DiffsWorker(Worker): def process_queue_item(self, job_details): image_id = job_details['image_id'] - namespace = job_details['namespace'] repository = job_details['repository'] + # TODO switch to the namespace_user_id branch only once exisiting jobs have all gone through + if 'namespace_user_id' in job_details: + namespace = model.get_namespace_by_user_id(job_details['namespace_user_id']) + else: + namespace = job_details['namespace'] + try: process_image_changes(namespace, repository, image_id) - except DataModelException: + except model.DataModelException: # This exception is unrecoverable, and the item should continue and be # marked as complete. msg = ('Image does not exist in database \'%s\' for repo \'%s/\'%s\'' % diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a45d82b67..870eeff95 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -38,7 +38,7 @@ TIMEOUT_PERIOD_MINUTES = 20 CACHE_EXPIRATION_PERIOD_HOURS = 24 NO_TAGS = [':'] RESERVATION_TIME = (TIMEOUT_PERIOD_MINUTES + 5) * 60 -DOCKER_BASE_URL = None # Set this if you want to use a different docker URL/socket. +DOCKER_BASE_URL = os.environ.get('DOCKER_HOST', None) def matches_system_error(status_str): @@ -130,8 +130,8 @@ class DockerfileBuildContext(object): # Note: We have two different clients here because we (potentially) login # with both, but with different credentials that we do not want shared between # the build and push operations. - self._push_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL) - self._build_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL) + self._push_cl = StreamingDockerClient(timeout=1200, base_url=DOCKER_BASE_URL) + self._build_cl = StreamingDockerClient(timeout=1200, base_url=DOCKER_BASE_URL) dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, 'Dockerfile') @@ -223,20 +223,6 @@ class DockerfileBuildContext(object): raise RuntimeError(message) def pull(self): - # Login with the specified credentials (if any). - if self._pull_credentials: - logger.debug('Logging in with pull credentials: %s@%s', - self._pull_credentials['username'], self._pull_credentials['registry']) - - self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { - 'phasestep': 'login', - 'username': self._pull_credentials['username'], - 'registry': self._pull_credentials['registry'] - }) - - self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], - registry=self._pull_credentials['registry'], reauth=True) - # Pull the image, in case it was updated since the last build image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag() if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: @@ -245,10 +231,24 @@ class DockerfileBuildContext(object): image_and_tag = ':'.join(image_and_tag_tuple) - self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { - 'phasestep': 'pull', - 'repo_url': image_and_tag - }) + # Login with the specified credentials (if any). + if self._pull_credentials: + logger.debug('Logging in with pull credentials: %s@%s', + self._pull_credentials['username'], self._pull_credentials['registry']) + + self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ + 'phasestep': 'login', + 'username': self._pull_credentials['username'], + 'registry': self._pull_credentials['registry'] + }) + + self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], + registry=self._pull_credentials['registry'], reauth=True) + else: + self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ + 'phasestep': 'pull', + 'repo_url': image_and_tag + }) pull_status = self._build_cl.pull(image_and_tag, stream=True) @@ -279,7 +279,7 @@ class DockerfileBuildContext(object): if key in status: fully_unwrapped = status[key] break - + if not fully_unwrapped: logger.debug('Status dict did not have any extractable keys and was: %s', status) elif isinstance(status, basestring): @@ -289,7 +289,7 @@ class DockerfileBuildContext(object): # Check for system errors when building. if matches_system_error(status_str): - raise WorkerUnhealthyException(status_str) + raise WorkerUnhealthyException(status_str) logger.debug('Status: %s', status_str) step_increment = re.search(r'Step ([0-9]+) :', status_str) @@ -481,8 +481,8 @@ class DockerfileBuildWorker(Worker): def watchdog(self): logger.debug('Running build watchdog code.') try: - docker_cl = Client(base_url = DOCKER_BASE_URL) - + docker_cl = Client(base_url=DOCKER_BASE_URL) + # Iterate the running containers and kill ones that have been running more than 20 minutes for container in docker_cl.containers(): start_time = datetime.fromtimestamp(container['Created']) @@ -502,9 +502,7 @@ class DockerfileBuildWorker(Worker): # Make sure we have more information for debugging problems sentry.client.user_context(job_details) - repository_build = model.get_repository_build(job_details['namespace'], - job_details['repository'], - job_details['build_uuid']) + repository_build = model.get_repository_build(job_details['build_uuid']) pull_credentials = job_details.get('pull_credentials', None) @@ -513,15 +511,21 @@ class DockerfileBuildWorker(Worker): resource_url = user_files.get_file_url(repository_build.resource_key, requires_cors=False) tag_names = job_config['docker_tags'] build_subdir = job_config['build_subdir'] - repo = job_config['repository'] + + # TODO remove the top branch when there are no more jobs with a repository config + if 'repository' in job_config: + repo = job_config['repository'] + else: + repo = '%s/%s/%s' % (job_config['registry'], + repository_build.repository.namespace_user.username, + repository_build.repository.name) access_token = repository_build.access_token.code - log_appender = partial(build_logs.append_log_message, - repository_build.uuid) + log_appender = partial(build_logs.append_log_message, repository_build.uuid) # Lookup and save the version of docker being used. - docker_cl = Client(base_url = DOCKER_BASE_URL) + docker_cl = Client(base_url=DOCKER_BASE_URL) docker_version = docker_cl.version().get('Version', '') dash = docker_version.find('-') @@ -529,14 +533,13 @@ class DockerfileBuildWorker(Worker): if dash > 0: docker_version = docker_version[:dash] - log_appender('initializing', build_logs.PHASE, log_data = { + log_appender('initializing', build_logs.PHASE, log_data={ 'docker_version': docker_version }) log_appender('Docker version: %s' % docker_version) - start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, - repo)) + start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, repo)) logger.debug(start_msg) docker_resource = requests.get(resource_url, stream=True) @@ -592,7 +595,7 @@ class DockerfileBuildWorker(Worker): cur_message = ex.message or 'Error while unpacking build package' log_appender(cur_message, build_logs.ERROR) spawn_failure(cur_message, event_data) - raise JobException(cur_message) + raise JobException(cur_message) # Start the build process. try: @@ -637,14 +640,14 @@ class DockerfileBuildWorker(Worker): # Spawn a notification that the build has completed. spawn_notification(repository_build.repository, 'build_success', event_data, - subpage='build?current=%s' % repository_build.uuid, - pathargs=['build', repository_build.uuid]) + subpage='build?current=%s' % repository_build.uuid, + pathargs=['build', repository_build.uuid]) except WorkerUnhealthyException as exc: # Spawn a notification that the build has failed. log_appender('Worker has become unhealthy. Will retry shortly.', build_logs.ERROR) spawn_failure(exc.message, event_data) - + # Raise the exception to the queue. raise exc diff --git a/workers/notificationworker.py b/workers/notificationworker.py index a176d46c8..e88428c5e 100644 --- a/workers/notificationworker.py +++ b/workers/notificationworker.py @@ -1,7 +1,4 @@ import logging -import argparse -import requests -import json from app import notification_queue from workers.worker import Worker @@ -12,11 +9,6 @@ from workers.worker import JobException from data import model -root_logger = logging.getLogger('') -root_logger.setLevel(logging.DEBUG) - -FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' -formatter = logging.Formatter(FORMAT) logger = logging.getLogger(__name__) @@ -24,10 +16,8 @@ logger = logging.getLogger(__name__) class NotificationWorker(Worker): def process_queue_item(self, job_details): notification_uuid = job_details['notification_uuid']; - repo_namespace = job_details['repository_namespace'] - repo_name = job_details['repository_name'] - notification = model.get_repo_notification(repo_namespace, repo_name, notification_uuid) + notification = model.get_repo_notification(notification_uuid) if not notification: # Probably deleted. return