From e8b3d1cc4a4d3de3fcdb0c291f1569b84cddfa8c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 1 Oct 2014 14:23:15 -0400 Subject: [PATCH 01/15] Phase 4 of the namespace to user migration: actually remove the column from the db and remove the dependence on serialized namespaces in the workers and queues --- ...translate_the_queue_names_to_reference_.py | 61 +++++++++++++ data/model/legacy.py | 52 +++++++---- data/queue.py | 59 +++++++------ endpoints/api/build.py | 10 ++- endpoints/api/repositorynotification.py | 16 +++- endpoints/api/user.py | 12 +++ endpoints/common.py | 6 +- endpoints/notificationhelper.py | 4 +- endpoints/registry.py | 10 ++- static/js/app.js | 1 + static/js/controllers.js | 22 ++++- static/partials/user-admin.html | 26 ++++++ test/data/test.db | Bin 630784 -> 626688 bytes test/testlogs.py | 6 +- workers/diffsworker.py | 18 ++-- workers/dockerfilebuild.py | 81 +++++++++--------- workers/notificationworker.py | 12 +-- 17 files changed, 273 insertions(+), 123 deletions(-) create mode 100644 data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py new file mode 100644 index 000000000..ae4c5d274 --- /dev/null +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -0,0 +1,61 @@ +"""Translate the queue names to reference namespace by id, remove the namespace column. + +Revision ID: 2fb36d4be80d +Revises: 3f4fe1194671 +Create Date: 2014-09-30 17:31:33.308490 + +""" + +# revision identifiers, used by Alembic. +revision = '2fb36d4be80d' +down_revision = '3f4fe1194671' + +from alembic import op +import sqlalchemy as sa + +import re +from app import app +from data.database import QueueItem, User, db + + +NAMESPACE_EXTRACTOR = re.compile(r'^([a-z]+/)([a-z0-9_]+)(/.*$)') + + +def upgrade(tables): + # Rename the namespace component of the existing queue items to reference user ids + with app.config['DB_TRANSACTION_FACTORY'](db): + for item in QueueItem.select(): + namespace_match = NAMESPACE_EXTRACTOR.match(item.queue_name) + if namespace_match is not None: + namespace_name = namespace_match.group(2) + namespace_user = User.get(User.username == namespace_name) + item.queue_name = '%s%s%s' % (namespace_match.group(1), str(namespace_user.id), + namespace_match.group(3)) + item.save() + else: + raise RuntimeError('Invalid queue name: %s' % item.queue_name) + + op.drop_index('repository_namespace_name', table_name='repository') + op.drop_column('repository', 'namespace') + + +def downgrade(tables): + # Add the namespace column back in and fill it in + op.add_column('repository', sa.Column('namespace', sa.String(length=255))) + conn = op.get_bind() + conn.execute('update repository set namespace = (select username from user where user.id = repository.namespace_user_id) where namespace is NULL') + op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True) + + # Rename the namespace component of existing queue items to reference namespace strings + with app.config['DB_TRANSACTION_FACTORY'](db): + for item in QueueItem.select(): + namespace_match = NAMESPACE_EXTRACTOR.match(item.queue_name) + if namespace_match is not None: + namespace_id = namespace_match.group(2) + namespace_user = User.get(User.id == namespace_id) + item.queue_name = '%s%s%s' % (namespace_match.group(1), + str(namespace_user.username), + namespace_match.group(3)) + item.save() + else: + raise RuntimeError('Invalid queue name: %s' % item.queue_name) diff --git a/data/model/legacy.py b/data/model/legacy.py index ba1fdd81f..effa2c8ef 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -592,6 +592,13 @@ def get_user_by_id(user_db_id): return None +def get_namespace_by_user_id(namespace_user_db_id): + try: + return User.get(User.id == namespace_user_db_id, User.robot == False).username + except User.DoesNotExist: + raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id) + + def get_user_or_org_by_customer_id(customer_id): try: return User.get(User.stripe_id == customer_id) @@ -858,6 +865,15 @@ def change_password(user, new_password): delete_notifications_by_kind(user, 'password_required') +def change_username(user, new_username): + (username_valid, username_issue) = validate_username(new_username) + if not username_valid: + raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue)) + + user.username = new_username + user.save() + + def change_invoice_email(user, invoice_email): user.invoice_email = invoice_email user.save() @@ -1676,10 +1692,21 @@ def load_token_data(code): raise InvalidTokenException('Invalid delegate token code: %s' % code) -def get_repository_build(namespace_name, repository_name, build_uuid): +def _get_build_base_query(): + return (RepositoryBuild + .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService, Repository, + Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuild) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .join(BuildTriggerService, JOIN_LEFT_OUTER) + .order_by(RepositoryBuild.started.desc())) + + +def get_repository_build(build_uuid): try: - query = list_repository_builds(namespace_name, repository_name, 1) - return query.where(RepositoryBuild.uuid == build_uuid).get() + return _get_build_base_query().where(RepositoryBuild.uuid == build_uuid).get() except RepositoryBuild.DoesNotExist: msg = 'Unable to locate a build by id: %s' % build_uuid @@ -1688,15 +1715,8 @@ def get_repository_build(namespace_name, repository_name, build_uuid): def list_repository_builds(namespace_name, repository_name, limit, include_inactive=True): - query = (RepositoryBuild - .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .switch(RepositoryBuild) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .join(BuildTriggerService, JOIN_LEFT_OUTER) + query = (_get_build_base_query() .where(Repository.name == repository_name, Namespace.username == namespace_name) - .order_by(RepositoryBuild.started.desc()) .limit(limit)) if not include_inactive: @@ -1760,21 +1780,23 @@ def create_repo_notification(repo, event_name, method_name, config): config_json=json.dumps(config)) -def get_repo_notification(namespace_name, repository_name, uuid): +def get_repo_notification(uuid): try: return (RepositoryNotification .select(RepositoryNotification, Repository, Namespace) .join(Repository) .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .where(Namespace.username == namespace_name, Repository.name == repository_name, - RepositoryNotification.uuid == uuid) + .where(RepositoryNotification.uuid == uuid) .get()) except RepositoryNotification.DoesNotExist: raise InvalidNotificationException('No repository notification found with id: %s' % uuid) def delete_repo_notification(namespace_name, repository_name, uuid): - found = get_repo_notification(namespace_name, repository_name, uuid) + found = get_repo_notification(uuid) + if (found.repository.namespace_user.username != namespace_name or + found.repository.name != repository_name): + raise InvalidNotificationException('No repository notifiation found with id: %s' % uuid) found.delete_instance() return found diff --git a/data/queue.py b/data/queue.py index 79e645ebf..73287dad6 100644 --- a/data/queue.py +++ b/data/queue.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta from data.database import QueueItem, db +from util.morecollections import AttrDict MINIMUM_EXTENSION = timedelta(seconds=20) @@ -25,17 +26,17 @@ class WorkQueue(object): def _running_jobs(self, now, name_match_query): return (QueueItem - .select(QueueItem.queue_name) - .where(QueueItem.available == False, - QueueItem.processing_expires > now, - QueueItem.queue_name ** name_match_query)) + .select(QueueItem.queue_name) + .where(QueueItem.available == False, + QueueItem.processing_expires > now, + QueueItem.queue_name ** name_match_query)) def _available_jobs(self, now, name_match_query, running_query): return (QueueItem - .select() - .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now, - ((QueueItem.available == True) | (QueueItem.processing_expires <= now)), - QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query))) + .select() + .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now, + ((QueueItem.available == True) | (QueueItem.processing_expires <= now)), + QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query))) def _name_match_query(self): return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) @@ -49,7 +50,7 @@ class WorkQueue(object): name_match_query = self._name_match_query() running_query = self._running_jobs(now, name_match_query) - running_count =running_query.distinct().count() + running_count = running_query.distinct().count() avialable_query = self._available_jobs(now, name_match_query, running_query) available_count = avialable_query.select(QueueItem.queue_name).distinct().count() @@ -89,41 +90,49 @@ class WorkQueue(object): item = None try: - item = avail.order_by(QueueItem.id).get() - item.available = False - item.processing_expires = now + timedelta(seconds=processing_time) - item.retries_remaining -= 1 - item.save() + db_item = avail.order_by(QueueItem.id).get() + db_item.available = False + db_item.processing_expires = now + timedelta(seconds=processing_time) + db_item.retries_remaining -= 1 + db_item.save() + + item = AttrDict({ + 'id': db_item.id, + 'body': db_item.body, + }) self._currently_processing = True except QueueItem.DoesNotExist: self._currently_processing = False - pass + # Return a view of the queue item rather than an active db object return item def complete(self, completed_item): with self._transaction_factory(db): - completed_item.delete_instance() + completed_item_obj = QueueItem.get(QueueItem.id == completed_item.id) + completed_item_obj.delete_instance() self._currently_processing = False def incomplete(self, incomplete_item, retry_after=300, restore_retry=False): with self._transaction_factory(db): retry_date = datetime.utcnow() + timedelta(seconds=retry_after) - incomplete_item.available_after = retry_date - incomplete_item.available = True + incomplete_item_obj = QueueItem.get(QueueItem.id == incomplete_item.id) + incomplete_item_obj.available_after = retry_date + incomplete_item_obj.available = True if restore_retry: - incomplete_item.retries_remaining += 1 + incomplete_item_obj.retries_remaining += 1 - incomplete_item.save() + incomplete_item_obj.save() self._currently_processing = False - @staticmethod - def extend_processing(queue_item, seconds_from_now): + def extend_processing(self, queue_item, seconds_from_now): new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now) # Only actually write the new expiration to the db if it moves the expiration some minimum - if new_expiration - queue_item.processing_expires > MINIMUM_EXTENSION: - queue_item.processing_expires = new_expiration - queue_item.save() + queue_item_obj = QueueItem.get(QueueItem.id == queue_item.id) + if new_expiration - queue_item_obj.processing_expires > MINIMUM_EXTENSION: + with self._transaction_factory(db): + queue_item_obj.processing_expires = new_expiration + queue_item_obj.save() diff --git a/endpoints/api/build.py b/endpoints/api/build.py index adf6f43ec..33398c15c 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -196,8 +196,9 @@ class RepositoryBuildStatus(RepositoryParamResource): @nickname('getRepoBuildStatus') def get(self, namespace, repository, build_uuid): """ Return the status for the builds specified by the build uuids. """ - build = model.get_repository_build(namespace, repository, build_uuid) - if not build: + build = model.get_repository_build(build_uuid) + if (not build or build.repository.name != repository or + build.repository.namespace_user.username != namespace): raise NotFound() can_write = ModifyRepositoryPermission(namespace, repository).can() @@ -213,7 +214,10 @@ class RepositoryBuildLogs(RepositoryParamResource): """ Return the build logs for the build specified by the build uuid. """ response_obj = {} - build = model.get_repository_build(namespace, repository, build_uuid) + build = model.get_repository_build(build_uuid) + if (not build or build.repository.name != repository or + build.repository.namespace_user.username != namespace): + raise NotFound() # If the logs have been archived, just redirect to the completed archive if build.logs_archived: diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 1fab89dd0..cc9bcb848 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -102,10 +102,14 @@ class RepositoryNotification(RepositoryParamResource): def get(self, namespace, repository, uuid): """ Get information for the specified notification. """ try: - notification = model.get_repo_notification(namespace, repository, uuid) + notification = model.get_repo_notification(uuid) except model.InvalidNotificationException: raise NotFound() + if (notification.repository.namespace_user.username != namespace or + notification.repository.name != repository): + raise NotFound() + return notification_view(notification) @require_repo_admin @@ -129,14 +133,18 @@ class TestRepositoryNotification(RepositoryParamResource): def post(self, namespace, repository, uuid): """ Queues a test notification for this repository. """ try: - notification = model.get_repo_notification(namespace, repository, uuid) + notification = model.get_repo_notification(uuid) except model.InvalidNotificationException: raise NotFound() + if (notification.repository.namespace_user.username != namespace or + notification.repository.name != repository): + raise NotFound() + event_info = NotificationEvent.get_event(notification.event.name) sample_data = event_info.get_sample_data(repository=notification.repository) notification_data = build_notification_data(notification, sample_data) - notification_queue.put([namespace, repository, notification.event.name], - json.dumps(notification_data)) + notification_queue.put([str(notification.repository.namespace_user.id), repository, + notification.event.name], json.dumps(notification_data)) return {} diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 7747addcc..b842bb466 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -139,6 +139,10 @@ class User(ApiResource): 'type': 'string', 'description': 'The user\'s email address', }, + 'username': { + 'type': 'string', + 'description': 'The user\'s username', + }, }, }, } @@ -189,6 +193,14 @@ class User(ApiResource): send_change_email(user.username, user_data['email'], code.code) else: model.update_email(user, new_email, auto_verify=not features.MAILING) + + if 'username' in user_data and user_data['username'] != user.username: + new_username = user_data['username'] + if model.get_user_or_org(new_username) is not None: + # Username already used + raise request_error(message='Username is already in use') + + model.change_username(user, new_username) except model.InvalidPasswordException, ex: raise request_error(exception=ex) diff --git a/endpoints/common.py b/endpoints/common.py index 37ae80ee8..c96a19c1d 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -213,7 +213,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, job_config = { 'docker_tags': tags, - 'repository': repo_path, + 'registry': host, 'build_subdir': subdir } @@ -221,10 +221,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, dockerfile_id, build_name, trigger, pull_robot_name=pull_robot_name) - dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({ + dockerfile_build_queue.put([str(repository.namespace_user.id), repository.name], json.dumps({ 'build_uuid': build_request.uuid, - 'namespace': repository.namespace_user.username, - 'repository': repository.name, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 6f80f83d0..cde307c30 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -30,8 +30,6 @@ def build_event_data(repo, extra_data={}, subpage=None): def build_notification_data(notification, event_data): return { 'notification_uuid': notification.uuid, - 'repository_namespace': notification.repository.namespace_user.username, - 'repository_name': notification.repository.name, 'event_data': event_data } @@ -43,5 +41,5 @@ def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[ event_name=event_name) for notification in notifications: notification_data = build_notification_data(notification, event_data) - path = [repo.namespace_user.username, repo.name, event_name] + pathargs + path = [str(repo.namespace_user.id), repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/registry.py b/endpoints/registry.py index 14bd88ce0..741601d0b 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -255,8 +255,9 @@ def put_image_layer(namespace, repository, image_id): # The layer is ready for download, send a job to the work queue to # process it. profile.debug('Adding layer to diff queue') - image_diff_queue.put([namespace, repository, image_id], json.dumps({ - 'namespace': namespace, + repo = model.get_repository(namespace, repository) + image_diff_queue.put([str(repo.namespace_user.id), repository, image_id], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, 'repository': repository, 'image_id': image_id, })) @@ -313,8 +314,9 @@ def put_image_checksum(namespace, repository, image_id): # The layer is ready for download, send a job to the work queue to # process it. profile.debug('Adding layer to diff queue') - image_diff_queue.put([namespace, repository, image_id], json.dumps({ - 'namespace': namespace, + repo = model.get_repository(namespace, repository) + image_diff_queue.put([str(repo.namespace_user.id), repository, image_id], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, 'repository': repository, 'image_id': image_id, })) diff --git a/static/js/app.js b/static/js/app.js index 27b086bab..adb11d6a9 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,6 @@ var TEAM_PATTERN = '^[a-zA-Z][a-zA-Z0-9]+$'; var ROBOT_PATTERN = '^[a-zA-Z][a-zA-Z0-9]{3,29}$'; +var USER_PATTERN = '^[a-z0-9_]{4,30}$'; $.fn.clipboardCopy = function() { if (zeroClipboardSupported) { diff --git a/static/js/controllers.js b/static/js/controllers.js index 908918631..566ce1f3a 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1676,6 +1676,8 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.logsShown = 0; $scope.invoicesShown = 0; + + $scope.USER_PATTERN = USER_PATTERN; $scope.loadAuthedApps = function() { if ($scope.authorizedApps) { return; } @@ -1752,6 +1754,24 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use }); }; + $scope.changeUsername = function() { + UserService.load(); + + $scope.updatingUser = true; + + ApiService.changeUserDetails($scope.cuser).then(function() { + $scope.updatingUser = false; + + // Reset the form. + delete $scope.cuser['username']; + + $scope.changeUsernameForm.$setPristine(); + }, function(result) { + $scope.updatingUser = false; + UIService.showFormError('#changeUsernameForm', result); + }); + }; + $scope.changeEmail = function() { UIService.hidePopover('#changeEmailForm'); @@ -1764,7 +1784,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.sentEmail = $scope.cuser.email; // Reset the form. - delete $scope.cuser['repeatEmail']; + delete $scope.cuser['email']; $scope.changeEmailForm.$setPristine(); }, function(result) { diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index a51119531..5196eb0bc 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -38,6 +38,7 @@
  • Usage Logs
    • +
  • Change Username
  • Convert to Organization
    • @@ -234,6 +235,31 @@
    + +
    +
    +
    +
    Change Username
    + +
    + +
    + + Username changed successfully + +
    +
    + + +
    +
    +
    +
    +
    +
    diff --git a/test/data/test.db b/test/data/test.db index af85d23e2016a9254ab3ad3b01876942f8d50c4f..88e3bd7a3b74e6be301c5d973a85d520468928ce 100644 GIT binary patch delta 8860 zcmeHNdwdghw$CK<$h1vTgwg`GO-rE^I>}7tl>vDq={s%HHcel&lu0rvrH}Oef>H*2 ztk>%^E5B8D!FBiUUR?oqb=y_$b#=YFy6Y;ai?~olT@~fwswiAhg@e@F^DsDso8FEJ zHvh7#-$ZnF!P`}VM~|A)S0ndC61tu-S@G103-r6(PHmO?u?TTh)}* z_^ELcSswXq_W+t+Rcwn$J=;+2*4KU*ne5q$T2AF=MK<&{poU{0 z5Si>PLuc&om>!|}0J>!V^SdL@^$nOZzdA6fnv>3&r6L=;tdaHoiYe=@Plna>8E3~A z#r#Me2%(naCMI&zK!ai0mm6O7TyxIP)d{3Ek~)}##%*5zU_=-MXzr$mwnttX96*zk z#w#Q4p>3${?c%#59}aCrb4s`RBXhSE8_*nAG8H+w9!6JPc}JvZ6rd@0 z{Ay+7Z=(ZfYH&$uq-n!8H0?OB+%>ip$CGYuYD6M~U0md~vE)QB4B@|fCA{3R6kcxM zd}eIo%oySsY2XU#3q1t|g5+~o*Dy?Jacz-QTP5<2LZ8i7C^+kiyybZnwI#)_T32~q zm6xt6s?GPh^Cgz%N{YD>8_jSOLsK+kX6%b;dNI#g?2?V81@P~ukjw_25i4v?uVm*N z$_ktg##7l)QeQ1NZ24?Koj1S2Rc5bp^Z9&bz2K>JaD`51owI_K7_Uq8)D)H1T+zNL zasmyqzCe~IUAnX&&czOM6$Hx2VO{-o=;s zJ#w{N)9nlo3w2U|n_?U2tL+TiR`jirJKK6KfpEv=tJ^GgL9{XKpx&UFrD8dxMh`!Z z9z_l+Jy|w6i?L-nN1W}wuG$g1Uv87yYusx}N(Mcp3R5^->-Af!huJ#!kh8x_unt=- zMFouH3Ho~MN@tk)2P^Y}$g(u({1P!Yl{uVsOlNT(=ZC&`c=F5E`n*<8Wm#cyYl&p_ zxa-%HcZZ!!M|<@^4d*WF?XPr7t({^1A1q-LM7v!AN1{uFYJFZ`NvMyOl+tRKv)gKM z_2+x)T^5_8XQS?^HLmUoe+^gN6V7*4)%LpF{arTjG*lhCC|wx~%iCF&AJiH<wk^ zHF^@}QcxM_ll|?=qF`T7*dOj=D~h-}##ZHlo21Y~d-F>i&hlcfSj4-_>Lo`}l_=KM zl$TX|%kuM!m_kpzjW4oQ=G&c4N2$XndM^OUbaszMSGL(;Jnd(YbiU19YOVFzi~V+s ztAt~_7>BQ>-r}xf?2gXxfQ=T``n|pOZeb{2a8!uBYZ;eozy+Q*8Z55 zG!qXPS>v!NVKAw0PY=kJ;G8Cor_8J!=iqT`hFb7;@H_ zR`(6K<$UMNp7^R+8*e5aH@;=cF?ByW&HR#atPq27Bf6_iIP|QYe-^&6OADCptf8w# zPFoNSjQl3h&S94E508WKJG5VvFDR*RaMpUAzPyq;p`e1UDlE;2@a6`T zAA_BaC(vh(`gyM9Va*f`Kbg7a%)W4QQ0`MQ7n?I{m0+&9)Xq6n$+A*6~q^im+iFd<%~LZN_2aY+KtQT`BwG|0&T#U2oX0Y(N7j-k$Tr}5Xe zY&1`gZ$xpk^8*d?f%ZLJxo#YFU@zcH@kfcd#J`Zw#)ackC3`3&u%V!TRUAB3a}ci&^tWRy6@4=}Y)VkJ z`xQ3eA)(3~#N`7A+p#Q42}!}AYzt7b5`ZeOi>CY%YokP)tOQvuNHcbStsXj;8eK)| zz;Y=?^vf`6hGGLu0Kz2M;VPn^l3=iYhGXrrpfnOtWfmg)zz)F;c$EOh|qe&9f~>nWR5;sWEO5sDZo*o;8b{= z1UG9yfmT9(S!6kd4un==&`Ji`x z!EnbEZs7M}EIBJ6`C&?iU=9e-TTT+8x&WCDiUEifV^{17Ki-Am<_Y|M?3DFqZ(@nG z0q*{3+HQ@uTYE&;gIUwe6^jSk_hAGm-izgcxBh}@UEG`ag#Yz8QJ z6&jj)H15N4U~s3Sfxsn`K;jWRW(JWXZ|Rc2hEIvO^B#GCxDPr5HFLnGeK0_HmjpUK zBN9Q{K5Qn~{u(yJ&_pCl5^`gBs^tgrBpiR|HEbplh(eeG+9t6?ux}4G7tDDdUommx zU$F%UvI^|n51|TzUmn3zz}5pWo@)+ZxkxiuegG=~NB3j%!7mPA%aGOJ@Bz#TZa4r3 z43iiMR!(A+UM7;=5~3~vA0NQ31s_d9Bp-{en>z`o!AB19gL$uGnaG-nqSrA6fxhf` z1ItEQr$RjP2DWJYBkZT!R=NxAp1J~`yS}8dCeP=nwlgJS9bL@Y>Ljkd)=5|R8YEA3 zO@*tp%Hu67cNIvU`i64aCsg|E&RTb#{_3iBm$g`UDFy-fJy9U7U2_k#WrlJudIXPK-ea94$tsQcU zGIi0Y6D$6kpb@Qq7(8>5n2&4(hffkqh^8StnMy^#nv+BtNIprVfNdv+>W%zQtj`Da4c$+0Q6ya zdE!m{Zxe^!##<2N>Rp-(9Pi@uqt3ytcNB*k=j=M@tgim{_U5TCy3MV@%p5g+Es^Yj z>FENKi-2oY~y?QOz>gU0(`XV+lb^QIHNf>BjpgD4Ys@oh45|~`0Gn>$BZ4l zAQlvCIfN&{FdsgIFGPB$qMd-zZr18Mh-7{dqME>uqfXs^7@v;xMbS-*Zm1biGbDK9 zFs?^BLHtqZ&9Wmf0J!N*;DN(e*?Zg5=ZzoXj?!!dZH%x;1 zvw?j(vFsb<&`1t=Y&((k&2q;y(R{XVzR<1NpkxQJ;2N87nQ+Rnhj$PQuOxhb2jRWO z#$Toc9ZdrMorL8|!UuK|(iO%hj_o9-uXs$qCGFe~KCwploSO-0ItOz3;df3P|Lx+q zZx@pnilg6FF2P8Dq5N_&ab8SIls&fn3`i0NR%SHv5-G*t__PZNtl>S;nBd0%Hd+gE%-6e0+C|4V`g&BuvUkp2l_ zi1ib-r-_-m0aXL6$6>AcUd(}^OUH>7fPMf;1&@D0{7R>$Zvc0m zB6#FR(EW2{HhAC^tV3r`5sX$D1v#gPdB_dGbBb85UAZ3o`Ew!z-1Rx!2wEu6sk&}( z?@^owuA}f&`|v2vAU(i*3>U%f=)iTy@C2~w7@h-<+J}$f*CHbme}qD!yASm@3iDa%TYk{1EPe4LB^@noS_`IU$a((8rsRA50{y z);F7w?}PTR-i<^ieiPQmSIjD6ii+w5XQd#x9rYYjxCp^D2v3pPsh{x@JvH@wak*Gi zTjR9Z>{W&1Pw5Ys=3T0!ovx$E%ckzp?>7Jc%d!8Pmt)|yGx{ua&bBi0F_J>iIBR=8 zU=Xs*GXWz<2-$GL0350CM=d9S}99kTCx|;)DExCeN_RKBMt7a*3T@Md- zYsnU3daIJHrrRS2;Xsd$4B_UVrXFb&BO7};a1Bak;o_X1c6x-!8}PXcC6kGT&o*sV zQ}_to$Af23xJ23-!a2l6{?x~TMvPp6CWHBJgZnXZn<)id_!l+ZcHzrPD;UMeAfAX;Z{L>lD5p9!)Vop-zH1y(&d zK+M{8`>m>(ij)uW;4M9jBy-Pmzf#k4BOk+o>q#<%&fDQ_0w0oO1HN!p#xc*r$VbB* z*c3-5qxh+3S@5Gc65w}jk`*2 zYV^g(t%gq1d$+WBs$=9a`mj-5>|+%8upW||9{ z-cD+Hn;=ee?4vKLDUKMqP0;m}xliN($13Q$;l+|8s(E>g+*Od=?EPye z!K!9Rt}W@e<>RZ#rHM!ySk^^;WSE{iJlivL4jK421-YQ5n@lm$*KN0`DT5aGCkPh6TZ&%Izka-Tg*h8)%a{hb9&(!oj$UFzSd&w4@ zntf(LV=rW$1=IS-c{soO*V8>cka-q_`^Xd`0ozEcDcz8H4*af?a+l_#V0i+}{tGH-EAJHZ{HTYv!YVj|{*?u+6{T(bxf*XTj`2a-KoE%0AQ6el9-v zw+OjlXb>)@{p8dkHKh$Q&w=L$$<=7a))y3D9g4!|c;^+>+zOfJz?(xbk_ErJ0dDIx zka-Sl9)@@_&wX44J{*S3&z`aI_nr`Bo(1X%nSw5Q{jeSUXaq99BzN;&L%|sH>mc*l z|M@qkY7Sh?{5r_|lKZ{{YPvth{Cdbd)AB-2qa0&?6f(c`2FDGa)iLHrA@d2>4#A{t zjxj$9na?VEW;%FJ!j%|5gvp54S zP$&=(C5-Mzn;F& zvz&9!a?kIabMMS}%g!ZRb}!kwaMo&_PWL1DH}$*mjvNjWsdv&bNWx9;gcw_X(LG=? zcJ;%{wfgTb95~+ zxOLDHT-$Rky6CXuFTv+~2GPjCGhYUk-f?vPbtD};(z^rAO1k;x;O0ITy3qLu8a&ch ziO%`(wKGAgAE5JBxg&$W?;kWRfAIFbnmO*gS}M4?+Y;P3pqh-96JuKP-1GG8upfzo zepLA4?k|Hk4!WY}JYBcWK6}c~bzc*fVB%05YJBqDmxJOEG_l~3Rl(PW22p8Ge_616 zcpQyv9(X5sVt5C-Vz>3<;MNfrntso(nu8}sD$(rCpX?3hjRG|Hou;_pq0vFpN^p3M zW+tZ{3(fbo4Js<_IFb-NxnT#o{6F9J1n=GGLYKC`HxWF!u@X(MTUi<`*#yw22k!n! z@ZC*=XvB@_w&1m!$5DP3QFiCp4m?3d?r%bZL*0Dvz*v0jJtHvu+t$Ft9V_5r>-A^H zCeMx`_EA@3MOjs`)n+SF3awT7Y(q)CWaR~xsVHAvXDyH#t2w!$v9zwSq@uEfV=HBN zl(=d!pDW=V`OZ2AOEWx0D-_F{S!S6aE@RnTL6B*I0grx#q`E|>gLODu4b@Chbv`RW zO$~KoQJswur9xYgy@W4Q_=;+_geeoMtLp7yQGK<|Nf+{Ltx%TFlt@?9&oi>Zg0xde zUfiXoS%K#`FnkIzkJYvnws%RQqBh9h_4PGv_7P9RK(58wZmXyqp;?R4&Pe@%Kt-!p zE^ta-m()_$UL5FRSM|5L1MPje-mcEet<$-JB8UPvL`3W6YgiVkBjC^4tGaz^ZyM`P zW7xEUhW3)KK)uCV(Ve@Vu4(a?RMqfZ1EY0Pqfc%r5}me+&RmDwQ`0k0+EZIuE>y6i zRRw`A=3iK5Wkp~(5cnD~Uz<<6tNRN4?7&c4QGr}oTTsJ|Sk%F;!C_baXs+gj^-0$u#SFd@nUqbQ*DG?MP3>k4d2AzNHr zLDLoGPNurXDGB+4v&z1@th~IuuG~>YJ8N9|C5=*DeNk~id84DGp&`Fav^$*@&YJwo z@PShhWm;ru@WE-sER2dn6|9x5V}0@nCrXlq8E~m~S9Pvu&{Im zM|H#chHAQ2_V^tA^|pSuMaeI(7+PIcSPx!+qGL1D6>6m=fm4(reYB3!Y0-p^AYm@~ z)WLqYr$f#5_4jspy86o-EA3^qm5OZTX|bT*;ap9Zm)WbC0+1SM$7;4j>*M=FcHgr%Ka$cF< z@C|Xx`p%g>d8Sz(X*TwlvP{p!y#32r=GS7ziZM8E#Bj9>n~GV=zibB+TlJ@^GE0OChs|DH#W6O8ZE!hwd$rKuDzOPw4CAaRaJZ`K z3g9YV1D-#wPX{H(_4Z8J&Qyps^lH{sQd8%suPJNf*-C+Pa`}y{Q!3$VYXs-&Mr-LL z6#2+fEeUSU?_(s2_DTxHdjyG+8C9VCqQEO~U8g11->Y_aEpz)i+XDS+Uw?mByBe78 zR)U$Pm+{;(I<%o+8IhidH5@UeeHXRc%m$i_xzBWu?t5^z10FCNPJ`|&0}94d4ZaJv zH^SS=@l-UiDUEllenyo!O7@8!ikE#H<#E#Zgp;9HG4V+;9EL`B3`kq<`YL_HHd7JWL#Yr4U-4({o)`TiK#`ZLr6dS68rOn&$?^xarKyui)y zAv7O-qQOdDdAjFZ+>QRyRQX*X&}SH&*f1kho)P^t12e$KdMRFHMasi^Rf-cNkJl%_ZG_jY zH&~E!0*DR@sBV=J1cvha1)k#F(1Ogu9kEySdp$-)_IVV)Ghvq1jbBh!# zJr&lW+egW~pP?mQP~5z?kqikhL<(jIXLVv}A`G7_czGY?7Q9f{D~Yf|+ziEueyGV$ z^SsZ$jtF_42rW-}4n{k%9GIAr>`|et#4-W}_by(_*y2fTyF)3x)z1Hzs&} zeo^*wq0rLMXXt_AW}xGu;FknOR8_&7hlYGknSoRbjJ7+_Kj04NSw)t;tm>w`Fuf?A z^>`>ZF9?*I@v%JPX8m3_-);ygSctSwPZ~aIba$x3dJx}*S;iRA$0)F{DL;IUqIjO= zDMjJ^6wPvsA~B*DMz#_rZ76Wew2r>EkPdzAxsy56S^L`hyLv}*d4cyp9AEU3w35xq`-lOYp5j&#*ti#40Cw!f6mWbmmJ%IWiURIVH8o31Tnz^HViK|jXsJXp_~Txz1eww`33X`* z(g5z*kEMXWzkw|T_r8gxg|3X?w{K!eQT?jBv$@R&^~5xWM{Rx+j-vNt>0sh(|#yx+K)wpvpEC~qW0nuVAlaG28`{;W=7&$*wP78gto|SjIUG> zEf4HG3_V+Z6eG7C#pZ#hLSpbvBs|9Waya0c@Bqk6LRvw}UMvCp<1o~E=qSX-3rxqb z)abT?8k@DwX0Eg5SJ=SrY+^1L`8{Hu>^+91Bhcr2j$;|f`rzA!IPk`CEL*=(1xfpG z8oaa*Uj)|g!x_X6?%#(mMLK|eAHER0vky;uWH%POZMPAfMBl=8j<0c)RoQH-xr$=W z&dFjiBjj`WWrC|%ax%2DkhKa@u~^PJM6SNHve=dHvK3d_Yila3N|{(y#g^3&moLU_ zF0aS}&BFIW_zR|0S8HKnEB+%(8Y9a-)knK2nuG0(=NLbw$Zpus1;5W9dJW(al*#9w z$BSY!W037SWII}cp2Qx;9r!24$0B+n-iz#uBBDTaV)SD%6Q*R-ZMqg%4jG2;=d=Lw z4O|9i_TvP2dJi@W+JYY*XN zNZ$<2uN=aQh)!dCb~bXe6~w-Q&js{h+=TRpv$x?ee!U*La^xs(0XH4PlflwscskMr z9LL}U=O2fJj$<=?Gl7?n;c-YeIDQOY1|ARbPaVTep!hhRgY<-?7J;qD;o;#-Snu#E z8r*ykj{|={j?Xb@hF;K}NeF<-g3KRg5)t5)Od|i@vzt4 zwBg(fiH@!oHPGKXGIicCIi5vG`iZx2vg+c^C)~2bHD6XhQm%3TWh|H#5(6SES7M2@ zB2Rv9F%Gm}EXL(Zw5ZSw*m9roPWyMstJF<(kr8lHr$cmBT?-j;g$?MmQuqDFf^V>Q zY`ZZN(A$kD`1Ae7v~L&$PmWa~8@*pm)Z`KEJa1!9*FnW$E+8NtuYL@jcA$czQ-GZS;puTH5@ zuqB09h~VJ*6rvKj4ZM~@wCGy~z=3Qc1FX*>5 z{2IOlxh||j$|aT{8^G8eJRZ4z670cgMBlIxJZd4#fXgJ}z)=gac)^-YP*c2qtS#VE zhub>cEoyTg+~L79_+18(1WGcAbiKM6{4|3|M{WSGWe|A=EqM$?WWadd2zq{kEC8+y z!j4`iT^_8HhC%I;m}a3GrhO+k=Al}Wc4a_UT>*VPVo1?2usoG01bb2l0^F2JI49mR zZc94zaX7Vp*8!uvCR>SyA_T^QK zUBVu_#5j5p|FU{Ed^`K1=kCk9LFg-;-t;*<>VWB#>GPOPF~6Eayl=vT4Ulx_A25+%@@hGZE>SUoMM6O=Dp}L5;%6rA&S{g6j~0BK7Cq zX2!bs0#|8CZK=&ENK!dlKJjznuqpjgA^jW!MP4>^H}R_Z|GyFb-+Utk2mV2%p|SIP zof99y{)R5@U-LO&J|-$nN^n=KmT=9Cudx((_G6+M=TB7XhL;6j=;1-_C&Vga&LZwE z&8!5M_X^Z@8&zeA!6x`CsgV~=FX}IO++Ge{LJO=N( zKPBRgS&x4aqa_GIx?cb!IAi+hU3M+O5>$o+ z@W+!xGfI5&Kpe1qK@1wBf@>!=GZm~F7QowIz?mppGoRCvbAlg3!uP%;{Ak*2xQLwi zl5pY4f?vY>TTa6` zrQdoA(GnI1%Qp$&*lFmx@!_;gU_Ap}zeXCdY37vRy3GRk=NX~}O+C0_BUpQu*nwt> zcRlnJNm)_kiBDDn#!6P27Ctg~Kuc)67+WjZjOz~k`p|Gg7~6bul`-$LeaV`+K8$TX z*K0 zt-Wd*+l~;n#blf@FS+QqT0&(Q+hVdA%r=&O}}~^nsRK z7RI)e^rP9CNd!1iO1kirvAN^+k}$SqWISrB-_{AXmyrNpw(F+F!$o0i%gI$HJn^%q zG_!36wiNhFInX&$rWCEei++I7^kJ{?p&iK zSi{&>Le~>t2Pc8G3c8+9mHMb=&I@B(MYf>guazjU)=us~xeg1gC> z8@-1$^8iGg2d{X@R-th9 z@{tL~6<=p>(-L|h;yn0`k6ec@$V!_(+zk=uz#5gbqA5FY7ucyn#50R6gIaRn0^%Wm zNBz)9>e^#3Hg!V8Ik2FGOpl&DnDHBX$JD&wzhdNo;TC9Yo;nHBs~sZFga2wF*P*j_ zJ$VbTw8F4qy2l)vxeX%DgST4YOvoKMf6$U!A>ur^z758a<-az86KxRj%xe4Ld4^TpTM<*<}l)&5b-%L-~Cfy4M5lP9O4t2`Pwkz0qE(9 j2XDO#tnGq`FWdg~;6yj+N=CV?f4~aA9nw~QN+$ge8%4cb diff --git a/test/testlogs.py b/test/testlogs.py index cd4ea6c9d..a6574e331 100644 --- a/test/testlogs.py +++ b/test/testlogs.py @@ -67,8 +67,7 @@ class TestBuildLogs(RedisBuildLogs): (phase, status) = status_wrapper from data import model - build_obj = model.get_repository_build(self.namespace, self.repository, - self.test_build_id) + build_obj = model.get_repository_build(self.test_build_id) build_obj.phase = phase build_obj.save() @@ -88,8 +87,7 @@ class TestBuildLogs(RedisBuildLogs): total_commands = random.randint(5, 20) for command_num in range(1, total_commands + 1): command_weight = random.randint(50, 100) - script.append(self._generate_command(command_num, total_commands, - command_weight)) + script.append(self._generate_command(command_num, total_commands, command_weight)) # we want 0 logs some percent of the time num_logs = max(0, random.randint(-50, 400)) diff --git a/workers/diffsworker.py b/workers/diffsworker.py index 563c61352..3a6a8459a 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -1,30 +1,28 @@ import logging -import argparse from app import image_diff_queue -from data.model import DataModelException +from data import model from endpoints.registry import process_image_changes from workers.worker import Worker -root_logger = logging.getLogger('') -root_logger.setLevel(logging.DEBUG) - -FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' -formatter = logging.Formatter(FORMAT) - logger = logging.getLogger(__name__) class DiffsWorker(Worker): def process_queue_item(self, job_details): image_id = job_details['image_id'] - namespace = job_details['namespace'] repository = job_details['repository'] + # TODO switch to the namespace_user_id branch only once exisiting jobs have all gone through + if 'namespace_user_id' in job_details: + namespace = model.get_namespace_by_user_id(job_details['namespace_user_id']) + else: + namespace = job_details['namespace'] + try: process_image_changes(namespace, repository, image_id) - except DataModelException: + except model.DataModelException: # This exception is unrecoverable, and the item should continue and be # marked as complete. msg = ('Image does not exist in database \'%s\' for repo \'%s/\'%s\'' % diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a45d82b67..870eeff95 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -38,7 +38,7 @@ TIMEOUT_PERIOD_MINUTES = 20 CACHE_EXPIRATION_PERIOD_HOURS = 24 NO_TAGS = [':'] RESERVATION_TIME = (TIMEOUT_PERIOD_MINUTES + 5) * 60 -DOCKER_BASE_URL = None # Set this if you want to use a different docker URL/socket. +DOCKER_BASE_URL = os.environ.get('DOCKER_HOST', None) def matches_system_error(status_str): @@ -130,8 +130,8 @@ class DockerfileBuildContext(object): # Note: We have two different clients here because we (potentially) login # with both, but with different credentials that we do not want shared between # the build and push operations. - self._push_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL) - self._build_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL) + self._push_cl = StreamingDockerClient(timeout=1200, base_url=DOCKER_BASE_URL) + self._build_cl = StreamingDockerClient(timeout=1200, base_url=DOCKER_BASE_URL) dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, 'Dockerfile') @@ -223,20 +223,6 @@ class DockerfileBuildContext(object): raise RuntimeError(message) def pull(self): - # Login with the specified credentials (if any). - if self._pull_credentials: - logger.debug('Logging in with pull credentials: %s@%s', - self._pull_credentials['username'], self._pull_credentials['registry']) - - self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { - 'phasestep': 'login', - 'username': self._pull_credentials['username'], - 'registry': self._pull_credentials['registry'] - }) - - self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], - registry=self._pull_credentials['registry'], reauth=True) - # Pull the image, in case it was updated since the last build image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag() if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: @@ -245,10 +231,24 @@ class DockerfileBuildContext(object): image_and_tag = ':'.join(image_and_tag_tuple) - self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { - 'phasestep': 'pull', - 'repo_url': image_and_tag - }) + # Login with the specified credentials (if any). + if self._pull_credentials: + logger.debug('Logging in with pull credentials: %s@%s', + self._pull_credentials['username'], self._pull_credentials['registry']) + + self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ + 'phasestep': 'login', + 'username': self._pull_credentials['username'], + 'registry': self._pull_credentials['registry'] + }) + + self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], + registry=self._pull_credentials['registry'], reauth=True) + else: + self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ + 'phasestep': 'pull', + 'repo_url': image_and_tag + }) pull_status = self._build_cl.pull(image_and_tag, stream=True) @@ -279,7 +279,7 @@ class DockerfileBuildContext(object): if key in status: fully_unwrapped = status[key] break - + if not fully_unwrapped: logger.debug('Status dict did not have any extractable keys and was: %s', status) elif isinstance(status, basestring): @@ -289,7 +289,7 @@ class DockerfileBuildContext(object): # Check for system errors when building. if matches_system_error(status_str): - raise WorkerUnhealthyException(status_str) + raise WorkerUnhealthyException(status_str) logger.debug('Status: %s', status_str) step_increment = re.search(r'Step ([0-9]+) :', status_str) @@ -481,8 +481,8 @@ class DockerfileBuildWorker(Worker): def watchdog(self): logger.debug('Running build watchdog code.') try: - docker_cl = Client(base_url = DOCKER_BASE_URL) - + docker_cl = Client(base_url=DOCKER_BASE_URL) + # Iterate the running containers and kill ones that have been running more than 20 minutes for container in docker_cl.containers(): start_time = datetime.fromtimestamp(container['Created']) @@ -502,9 +502,7 @@ class DockerfileBuildWorker(Worker): # Make sure we have more information for debugging problems sentry.client.user_context(job_details) - repository_build = model.get_repository_build(job_details['namespace'], - job_details['repository'], - job_details['build_uuid']) + repository_build = model.get_repository_build(job_details['build_uuid']) pull_credentials = job_details.get('pull_credentials', None) @@ -513,15 +511,21 @@ class DockerfileBuildWorker(Worker): resource_url = user_files.get_file_url(repository_build.resource_key, requires_cors=False) tag_names = job_config['docker_tags'] build_subdir = job_config['build_subdir'] - repo = job_config['repository'] + + # TODO remove the top branch when there are no more jobs with a repository config + if 'repository' in job_config: + repo = job_config['repository'] + else: + repo = '%s/%s/%s' % (job_config['registry'], + repository_build.repository.namespace_user.username, + repository_build.repository.name) access_token = repository_build.access_token.code - log_appender = partial(build_logs.append_log_message, - repository_build.uuid) + log_appender = partial(build_logs.append_log_message, repository_build.uuid) # Lookup and save the version of docker being used. - docker_cl = Client(base_url = DOCKER_BASE_URL) + docker_cl = Client(base_url=DOCKER_BASE_URL) docker_version = docker_cl.version().get('Version', '') dash = docker_version.find('-') @@ -529,14 +533,13 @@ class DockerfileBuildWorker(Worker): if dash > 0: docker_version = docker_version[:dash] - log_appender('initializing', build_logs.PHASE, log_data = { + log_appender('initializing', build_logs.PHASE, log_data={ 'docker_version': docker_version }) log_appender('Docker version: %s' % docker_version) - start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, - repo)) + start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, repo)) logger.debug(start_msg) docker_resource = requests.get(resource_url, stream=True) @@ -592,7 +595,7 @@ class DockerfileBuildWorker(Worker): cur_message = ex.message or 'Error while unpacking build package' log_appender(cur_message, build_logs.ERROR) spawn_failure(cur_message, event_data) - raise JobException(cur_message) + raise JobException(cur_message) # Start the build process. try: @@ -637,14 +640,14 @@ class DockerfileBuildWorker(Worker): # Spawn a notification that the build has completed. spawn_notification(repository_build.repository, 'build_success', event_data, - subpage='build?current=%s' % repository_build.uuid, - pathargs=['build', repository_build.uuid]) + subpage='build?current=%s' % repository_build.uuid, + pathargs=['build', repository_build.uuid]) except WorkerUnhealthyException as exc: # Spawn a notification that the build has failed. log_appender('Worker has become unhealthy. Will retry shortly.', build_logs.ERROR) spawn_failure(exc.message, event_data) - + # Raise the exception to the queue. raise exc diff --git a/workers/notificationworker.py b/workers/notificationworker.py index a176d46c8..e88428c5e 100644 --- a/workers/notificationworker.py +++ b/workers/notificationworker.py @@ -1,7 +1,4 @@ import logging -import argparse -import requests -import json from app import notification_queue from workers.worker import Worker @@ -12,11 +9,6 @@ from workers.worker import JobException from data import model -root_logger = logging.getLogger('') -root_logger.setLevel(logging.DEBUG) - -FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' -formatter = logging.Formatter(FORMAT) logger = logging.getLogger(__name__) @@ -24,10 +16,8 @@ logger = logging.getLogger(__name__) class NotificationWorker(Worker): def process_queue_item(self, job_details): notification_uuid = job_details['notification_uuid']; - repo_namespace = job_details['repository_namespace'] - repo_name = job_details['repository_name'] - notification = model.get_repo_notification(repo_namespace, repo_name, notification_uuid) + notification = model.get_repo_notification(notification_uuid) if not notification: # Probably deleted. return From c77a3dc6eae0dd485de7b3cc34008679fe3cb85a Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 7 Oct 2014 15:57:17 -0400 Subject: [PATCH 02/15] Update the migration for translating queue names to take into account previous migrations. --- ...fb36d4be80d_translate_the_queue_names_to_reference_.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py index 53b43a4ec..bdeb3345f 100644 --- a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -1,6 +1,6 @@ """Translate the queue names to reference namespace by id, remove the namespace column. -Revision ID: 2fb36d4be80d +Revision ID: 2430f55c41d5 Revises: 9a1087b007d Create Date: 2014-09-30 17:31:33.308490 @@ -8,7 +8,7 @@ Create Date: 2014-09-30 17:31:33.308490 # revision identifiers, used by Alembic. revision = '2fb36d4be80d' -down_revision = '9a1087b007d' +down_revision = '2430f55c41d5' from alembic import op import sqlalchemy as sa @@ -35,16 +35,12 @@ def upgrade(tables): else: raise RuntimeError('Invalid queue name: %s' % item.queue_name) - op.drop_index('repository_namespace_name', table_name='repository') op.drop_column('repository', 'namespace') def downgrade(tables): # Add the namespace column back in and fill it in op.add_column('repository', sa.Column('namespace', sa.String(length=255))) - conn = op.get_bind() - conn.execute('update repository set namespace = (select username from user where user.id = repository.namespace_user_id) where namespace is NULL') - op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True) # Rename the namespace component of existing queue items to reference namespace strings with app.config['DB_TRANSACTION_FACTORY'](db): From bbcdf18fdd1af37c3291c3fd68a2b9f50c305fa6 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 7 Oct 2014 16:19:05 -0400 Subject: [PATCH 03/15] Update the migration to reflect the state of the database. --- .../2fb36d4be80d_translate_the_queue_names_to_reference_.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py index bdeb3345f..15b740cbf 100644 --- a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -35,12 +35,14 @@ def upgrade(tables): else: raise RuntimeError('Invalid queue name: %s' % item.queue_name) + op.create_index('repository_namespace_user_id', 'repository', ['namespace_user_id'], unique=False) op.drop_column('repository', 'namespace') def downgrade(tables): # Add the namespace column back in and fill it in op.add_column('repository', sa.Column('namespace', sa.String(length=255))) + op.drop_index('repository_namespace_user_id', table_name='repository') # Rename the namespace component of existing queue items to reference namespace strings with app.config['DB_TRANSACTION_FACTORY'](db): From ca435fc7a658abce6eb2102c7d237d8523e0d786 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Sun, 9 Nov 2014 17:50:57 -0500 Subject: [PATCH 04/15] Rename robots when we rename a user. Do not use the namespace from the path to check permissions from the incoming webhooks since the namespace may have changed and we cannot recreate them in remote services easily. --- data/model/legacy.py | 21 ++++++++++++++------- endpoints/webhooks.py | 24 +++++++++++++----------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/data/model/legacy.py b/data/model/legacy.py index 6466e4171..1be64f72c 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -18,7 +18,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor from peewee import JOIN_LEFT_OUTER, fn from util.validation import (validate_username, validate_email, validate_password, INVALID_PASSWORD_MESSAGE) -from util.names import format_robot_username +from util.names import format_robot_username, parse_robot_username from util.backoff import exponential_backoff @@ -878,8 +878,17 @@ def change_username(user, new_username): if not username_valid: raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue)) - user.username = new_username - user.save() + with config.app_config['DB_TRANSACTION_FACTORY'](db): + # Rename the robots + for robot in list_entity_robots(user.username): + _, robot_shortname = parse_robot_username(robot.username) + new_robot_name = format_robot_username(new_username, robot_shortname) + robot.username = new_robot_name + robot.save() + + # Rename the user + user.username = new_username + user.save() def change_invoice_email(user, invoice_email): @@ -1955,7 +1964,7 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None): return trigger -def get_build_trigger(namespace_name, repository_name, trigger_uuid): +def get_build_trigger(trigger_uuid): try: return (RepositoryBuildTrigger .select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace) @@ -1965,9 +1974,7 @@ def get_build_trigger(namespace_name, repository_name, trigger_uuid): .join(Namespace, on=(Repository.namespace_user == Namespace.id)) .switch(RepositoryBuildTrigger) .join(User) - .where(RepositoryBuildTrigger.uuid == trigger_uuid, - Namespace.username == namespace_name, - Repository.name == repository_name) + .where(RepositoryBuildTrigger.uuid == trigger_uuid) .get()) except RepositoryBuildTrigger.DoesNotExist: msg = 'No build trigger with uuid: %s' % trigger_uuid diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 0d9fcd227..20eb3689b 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -67,20 +67,22 @@ def stripe_webhook(): return make_response('Okay') -@webhooks.route('/push//trigger/', - methods=['POST']) +@webhooks.route('/push//trigger/', methods=['POST']) @process_auth -@parse_repository_name -def build_trigger_webhook(namespace, repository, trigger_uuid): - logger.debug('Webhook received for %s/%s with uuid %s', namespace, - repository, trigger_uuid) +def build_trigger_webhook(_, trigger_uuid): + logger.debug('Webhook received with uuid %s', trigger_uuid) + + try: + trigger = model.get_build_trigger(trigger_uuid) + except model.InvalidBuildTriggerException: + # It is ok to return 404 here, since letting an attacker know that a trigger UUID is valid + # doesn't leak anything + abort(404) + + namespace = trigger.repository.namespace_user.username + repository = trigger.repository.name permission = ModifyRepositoryPermission(namespace, repository) if permission.can(): - try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) - except model.InvalidBuildTriggerException: - abort(404) - handler = BuildTrigger.get_trigger_for_service(trigger.service.name) logger.debug('Passing webhook request to handler %s', handler) From a7bae6c1d9481f12fde866088021c25298d8ef22 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 18 Nov 2014 10:24:48 -0500 Subject: [PATCH 05/15] Fix robot renaming. Allow for trigger URLs to contain or omit the repository in the path. Fix calls to get_trigger to remove the namespace and repository. --- data/model/legacy.py | 18 ++++++++++++------ endpoints/api/robot.py | 4 ++-- endpoints/api/trigger.py | 16 ++++++++-------- endpoints/webhooks.py | 3 ++- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/data/model/legacy.py b/data/model/legacy.py index d42db7137..c06df82de 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -294,11 +294,17 @@ def delete_robot(robot_username): robot_username) -def list_entity_robots(entity_name): - selected = User.select(User.username, FederatedLogin.service_ident) - joined = selected.join(FederatedLogin) - return joined.where(User.robot == True, - User.username ** (entity_name + '+%')).tuples() +def _list_entity_robots(entity_name): + return (User + .select() + .join(FederatedLogin) + .where(User.robot == True, User.username ** (entity_name + '+%'))) + + +def list_entity_robot_tuples(entity_name): + return (_list_entity_robots(entity_name) + .select(User.username, FederatedLogin.service_ident) + .tuples()) def convert_user_to_organization(user, admin_user): @@ -893,7 +899,7 @@ def change_username(user, new_username): with config.app_config['DB_TRANSACTION_FACTORY'](db): # Rename the robots - for robot in list_entity_robots(user.username): + for robot in _list_entity_robots(user.username): _, robot_shortname = parse_robot_username(robot.username) new_robot_name = format_robot_username(new_username, robot_shortname) robot.username = new_robot_name diff --git a/endpoints/api/robot.py b/endpoints/api/robot.py index b52cd4c5b..b8338c911 100644 --- a/endpoints/api/robot.py +++ b/endpoints/api/robot.py @@ -24,7 +24,7 @@ class UserRobotList(ApiResource): def get(self): """ List the available robots for the user. """ user = get_authenticated_user() - robots = model.list_entity_robots(user.username) + robots = model.list_entity_robot_tuples(user.username) return { 'robots': [robot_view(name, password) for name, password in robots] } @@ -73,7 +73,7 @@ class OrgRobotList(ApiResource): """ List the organization's robots. """ permission = OrganizationMemberPermission(orgname) if permission.can(): - robots = model.list_entity_robots(orgname) + robots = model.list_entity_robot_tuples(orgname) return { 'robots': [robot_view(name, password) for name, password in robots] } diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index be119499d..49f67c26e 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -52,7 +52,7 @@ class BuildTrigger(RepositoryParamResource): def get(self, namespace, repository, trigger_uuid): """ Get information for the specified build trigger. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -63,7 +63,7 @@ class BuildTrigger(RepositoryParamResource): def delete(self, namespace, repository, trigger_uuid): """ Delete the specified build trigger. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -107,7 +107,7 @@ class BuildTriggerSubdirs(RepositoryParamResource): def post(self, namespace, repository, trigger_uuid): """ List the subdirectories available for the specified build trigger and source. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -167,7 +167,7 @@ class BuildTriggerActivate(RepositoryParamResource): def post(self, namespace, repository, trigger_uuid): """ Activate the specified build trigger. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -264,7 +264,7 @@ class BuildTriggerAnalyze(RepositoryParamResource): def post(self, namespace, repository, trigger_uuid): """ Analyze the specified build trigger configuration. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -395,7 +395,7 @@ class ActivateBuildTrigger(RepositoryParamResource): def post(self, namespace, repository, trigger_uuid): """ Manually start a build from the specified trigger. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -453,7 +453,7 @@ class BuildTriggerFieldValues(RepositoryParamResource): def post(self, namespace, repository, trigger_uuid, field_name): """ List the field values for a custom run field. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() @@ -482,7 +482,7 @@ class BuildTriggerSources(RepositoryParamResource): def get(self, namespace, repository, trigger_uuid): """ List the build sources for the trigger configuration thus far. """ try: - trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + trigger = model.get_build_trigger(trigger_uuid) except model.InvalidBuildTriggerException: raise NotFound() diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 20eb3689b..24d8edf3d 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -68,8 +68,9 @@ def stripe_webhook(): @webhooks.route('/push//trigger/', methods=['POST']) +@webhooks.route('/push/trigger/', methods=['POST'], defaults={'repository': ''}) @process_auth -def build_trigger_webhook(_, trigger_uuid): +def build_trigger_webhook(trigger_uuid, **kwargs): logger.debug('Webhook received with uuid %s', trigger_uuid) try: From 3815e9a29355d363ae7089fb5b295b5a7937279d Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 18 Nov 2014 10:29:59 -0500 Subject: [PATCH 06/15] Switch to installing paths in github which do not include repository information. --- endpoints/api/trigger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 49f67c26e..56bc061dc 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -205,10 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource): 'write') try: - repository_path = '%s/%s' % (trigger.repository.namespace_user.username, - trigger.repository.name) - path = url_for('webhooks.build_trigger_webhook', - repository=repository_path, trigger_uuid=trigger.uuid) + path = url_for('webhooks.build_trigger_webhook', trigger_uuid=trigger.uuid) authed_url = _prepare_webhook_url(app.config['PREFERRED_URL_SCHEME'], '$token', token.code, app.config['SERVER_HOSTNAME'], path) From 17fc72d2625ca7b178bf3cbc3b9c4a42f13ba4a2 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 18 Nov 2014 14:07:33 -0500 Subject: [PATCH 07/15] Switch postgres to a non-transactional DDL to allow us to use peewee to modify data in migrations: enterprise customers are running postgres migrations offline already. Move the image backfill script back to a migration since it will now work. Unify the interface to sending a DB URI to env.py for the migration script. --- conf/init/runmigration.sh | 5 +---- data/migrations/env.py | 21 +++--------------- data/migrations/migration.sh | 22 ++++++++++--------- ...5_calculate_uncompressed_sizes_for_all_.py | 4 +--- ...translate_the_queue_names_to_reference_.py | 4 ++-- util/uncompressedsize.py | 10 ++------- 6 files changed, 21 insertions(+), 45 deletions(-) diff --git a/conf/init/runmigration.sh b/conf/init/runmigration.sh index a82349dcf..8b006b745 100755 --- a/conf/init/runmigration.sh +++ b/conf/init/runmigration.sh @@ -2,7 +2,4 @@ set -e # Run the database migration -PYTHONPATH=. venv/bin/alembic upgrade head - -# Run the uncompressed size migration -PYTHONPATH=. venv/bin/python -m util.uncompressedsize \ No newline at end of file +PYTHONPATH=. venv/bin/alembic upgrade head \ No newline at end of file diff --git a/data/migrations/env.py b/data/migrations/env.py index 650683572..3b2df5186 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -13,24 +13,8 @@ from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata from util.morecollections import AttrDict -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -db_uri = unquote(app.config['DB_URI']) -if 'GENMIGRATE' in os.environ: - docker_host = os.environ.get('DOCKER_HOST') - docker_host_ip = docker_host[len('tcp://'):].split(':')[0] - if os.environ.get('GENMIGRATE') == 'mysql': - db_uri = 'mysql+pymysql://root:password@%s/genschema' % (docker_host_ip) - else: - db_uri = 'postgresql://postgres@%s/genschema' % (docker_host_ip) - -if 'DB_URI' in os.environ: - db_uri = os.environ['DB_URI'] - -app.config['DB_URI'] = db_uri - config = context.config -config.set_main_option('sqlalchemy.url', db_uri) +config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI'])) # Interpret the config file for Python logging. # This line sets up loggers basically. @@ -86,7 +70,8 @@ def run_migrations_online(): connection = engine.connect() context.configure( connection=connection, - target_metadata=target_metadata + target_metadata=target_metadata, + transactional_ddl=False, ) try: diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index 5deeba3dd..dedaf2445 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -1,4 +1,8 @@ -set -e +set -e + +DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'` +MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}" +PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}" up_mysql() { # Run a SQL database on port 3306 inside of Docker. @@ -36,19 +40,19 @@ down_postgres() { gen_migrate() { # Generate a SQLite database with the schema as defined by the existing alembic model. - GENMIGRATE=$1 PYTHONPATH=. alembic upgrade head + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head # Generate the migration to the current model. - GENMIGRATE=$1 PYTHONPATH=. alembic revision --autogenerate -m "$2" + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic revision --autogenerate -m "$2" } test_migrate() { # Generate a SQLite database with the schema as defined by the existing alembic model. - GENMIGRATE=$1 PYTHONPATH=. alembic upgrade head + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head # Downgrade to verify it works in both directions. COUNT=`ls data/migrations/versions/*.py | wc -l | tr -d ' '` - GENMIGRATE=$1 PYTHONPATH=. alembic downgrade "-$COUNT" + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT" } # Test (and generate, if requested) via MySQL. @@ -59,13 +63,13 @@ if [ ! -z "$@" ] then set +e echo '> Generating Migration' - gen_migrate "mysql" "$@" + gen_migrate $MYSQL_CONFIG_OVERRIDE "$@" set -e fi echo '> Testing Migration (mysql)' set +e -test_migrate "mysql" +test_migrate $MYSQL_CONFIG_OVERRIDE set -e down_mysql @@ -75,8 +79,6 @@ up_postgres echo '> Testing Migration (postgres)' set +e -test_migrate "postgres" +test_migrate $PGSQL_CONFIG_OVERRIDE set -e down_postgres - - diff --git a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py index 03f6d7733..df2ed6f14 100644 --- a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py +++ b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py @@ -16,9 +16,7 @@ from util.uncompressedsize import backfill_sizes_from_data def upgrade(tables): - # Note: Doing non-alembic operations inside alembic can cause a deadlock. This call has been - # moved to runmigration.sh. - pass + backfill_sizes_from_data() def downgrade(tables): pass diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py index 15b740cbf..76e698471 100644 --- a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -1,14 +1,14 @@ """Translate the queue names to reference namespace by id, remove the namespace column. Revision ID: 2430f55c41d5 -Revises: 9a1087b007d +Revises: 313d297811c4 Create Date: 2014-09-30 17:31:33.308490 """ # revision identifiers, used by Alembic. revision = '2fb36d4be80d' -down_revision = '2430f55c41d5' +down_revision = '313d297811c4' from alembic import op import sqlalchemy as sa diff --git a/util/uncompressedsize.py b/util/uncompressedsize.py index 5a6f16602..d39c43540 100644 --- a/util/uncompressedsize.py +++ b/util/uncompressedsize.py @@ -1,9 +1,8 @@ import logging import zlib -import sys from data import model -from data.database import ImageStorage, configure +from data.database import ImageStorage from app import app, storage as store from data.database import db, db_random_func from util.gzipstream import ZLIB_GZIP_WINDOW @@ -15,16 +14,11 @@ logger = logging.getLogger(__name__) CHUNK_SIZE = 5 * 1024 * 1024 def backfill_sizes_from_data(): - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) logger.debug('Starting uncompressed image size backfill') logger.debug('NOTE: This can be a LONG RUNNING OPERATION. Please wait!') - # Make sure we have a reference to the current DB. - configure(app.config) - - logger.debug('Uncompressed backfill: Database configured') - # Check for any uncompressed images. has_images = bool(list(ImageStorage .select(ImageStorage.uuid) From 9d677b8eb3c8ae996df4ba44bd3617fdb3462f69 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 11 Nov 2014 17:22:37 -0500 Subject: [PATCH 08/15] Add UUID to User model and use in cookie. --- auth/auth.py | 10 ++-- auth/auth_context.py | 14 +++--- auth/permissions.py | 18 +++---- data/database.py | 11 ++-- .../17f11e265e13_add_uuid_field_to_user.py | 26 ++++++++++ data/model/legacy.py | 47 ++++++++++-------- endpoints/common.py | 26 ++++------ test/data/test.db | Bin 684032 -> 251904 bytes 8 files changed, 91 insertions(+), 61 deletions(-) create mode 100644 data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py diff --git a/auth/auth.py b/auth/auth.py index ed0c8d82a..66ba4b921 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -25,7 +25,7 @@ def _load_user_from_cookie(): if not current_user.is_anonymous(): logger.debug('Loading user from cookie: %s', current_user.get_id()) set_authenticated_user_deferred(current_user.get_id()) - loaded = QuayDeferredPermissionUser(current_user.get_id(), 'user_db_id', {scopes.DIRECT_LOGIN}) + loaded = QuayDeferredPermissionUser(current_user.get_id(), 'user_uuid', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=loaded) return current_user.db_user() return None @@ -58,7 +58,7 @@ def _validate_and_apply_oauth_token(token): set_authenticated_user(validated.authorized_user) set_validated_oauth_token(validated) - new_identity = QuayDeferredPermissionUser(validated.authorized_user.id, 'user_db_id', scope_set) + new_identity = QuayDeferredPermissionUser(validated.authorized_user.uuid, 'user_uuid', scope_set) identity_changed.send(app, identity=new_identity) @@ -97,8 +97,8 @@ def process_basic_auth(auth): robot = model.verify_robot(credentials[0], credentials[1]) logger.debug('Successfully validated robot: %s' % credentials[0]) set_authenticated_user(robot) - - deferred_robot = QuayDeferredPermissionUser(robot.id, 'user_db_id', {scopes.DIRECT_LOGIN}) + + deferred_robot = QuayDeferredPermissionUser(robot.uuid, 'user_uuid', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=deferred_robot) return except model.InvalidRobotException: @@ -111,7 +111,7 @@ def process_basic_auth(auth): logger.debug('Successfully validated user: %s' % authenticated.username) set_authenticated_user(authenticated) - new_identity = QuayDeferredPermissionUser(authenticated.id, 'user_db_id', + new_identity = QuayDeferredPermissionUser(authenticated.uuid, 'user_uuid', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) return diff --git a/auth/auth_context.py b/auth/auth_context.py index 6c587f901..4f9cc4a3d 100644 --- a/auth/auth_context.py +++ b/auth/auth_context.py @@ -10,13 +10,13 @@ logger = logging.getLogger(__name__) def get_authenticated_user(): user = getattr(_request_ctx_stack.top, 'authenticated_user', None) if not user: - db_id = getattr(_request_ctx_stack.top, 'authenticated_db_id', None) - if not db_id: - logger.debug('No authenticated user or deferred database id.') + user_uuid = getattr(_request_ctx_stack.top, 'authenticated_user_uuid', None) + if not user_uuid: + logger.debug('No authenticated user or deferred database uuid.') return None logger.debug('Loading deferred authenticated user.') - loaded = model.get_user_by_id(db_id) + loaded = model.get_user_by_uuid(user_uuid) set_authenticated_user(loaded) user = loaded @@ -30,10 +30,10 @@ def set_authenticated_user(user_or_robot): ctx.authenticated_user = user_or_robot -def set_authenticated_user_deferred(user_or_robot_db_id): - logger.debug('Deferring loading of authenticated user object: %s', user_or_robot_db_id) +def set_authenticated_user_deferred(user_or_robot_uuid): + logger.debug('Deferring loading of authenticated user object with uuid: %s', user_or_robot_uuid) ctx = _request_ctx_stack.top - ctx.authenticated_db_id = user_or_robot_db_id + ctx.authenticated_user_uuid = user_or_robot_uuid def get_validated_oauth_token(): diff --git a/auth/permissions.py b/auth/permissions.py index eb9059c22..eee8d75ff 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -58,8 +58,8 @@ SCOPE_MAX_USER_ROLES.update({ class QuayDeferredPermissionUser(Identity): - def __init__(self, db_id, auth_type, scopes): - super(QuayDeferredPermissionUser, self).__init__(db_id, auth_type) + def __init__(self, id, auth_type, scopes): + super(QuayDeferredPermissionUser, self).__init__(id, auth_type) self._permissions_loaded = False self._scope_set = scopes @@ -88,14 +88,14 @@ class QuayDeferredPermissionUser(Identity): def can(self, permission): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') - user_object = model.get_user_by_id(self.id) + user_object = model.get_user_by_uuid(self.id) # Add the superuser need, if applicable. if (user_object.username is not None and user_object.username in app.config.get('SUPER_USERS', [])): self.provides.add(_SuperUserNeed()) - # Add the user specific permissions, only for non-oauth permission + # Add the user specific permissions, only for non-oauth permission user_grant = _UserNeed(user_object.username, self._user_role_for_scopes('admin')) logger.debug('User permission: {0}'.format(user_grant)) self.provides.add(user_grant) @@ -217,7 +217,7 @@ class ViewTeamPermission(Permission): team_admin = _TeamNeed(org_name, team_name, 'admin') team_creator = _TeamNeed(org_name, team_name, 'creator') team_member = _TeamNeed(org_name, team_name, 'member') - admin_org = _OrganizationNeed(org_name, 'admin') + admin_org = _OrganizationNeed(org_name, 'admin') super(ViewTeamPermission, self).__init__(team_admin, team_creator, team_member, admin_org) @@ -228,11 +228,11 @@ def on_identity_loaded(sender, identity): # We have verified an identity, load in all of the permissions if isinstance(identity, QuayDeferredPermissionUser): - logger.debug('Deferring permissions for user: %s', identity.id) + logger.debug('Deferring permissions for user with uuid: %s', identity.id) - elif identity.auth_type == 'user_db_id': - logger.debug('Switching username permission to deferred object: %s', identity.id) - switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'user_db_id', {scopes.DIRECT_LOGIN}) + elif identity.auth_type == 'user_uuid': + logger.debug('Switching username permission to deferred object with uuid: %s', identity.id) + switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'user_uuid', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=switch_to_deferred) elif identity.auth_type == 'token': diff --git a/data/database.py b/data/database.py index c3204f161..0ccdbf54f 100644 --- a/data/database.py +++ b/data/database.py @@ -26,7 +26,7 @@ SCHEME_RANDOM_FUNCTION = { 'mysql+pymysql': fn.Rand, 'sqlite': fn.Random, 'postgresql': fn.Random, - 'postgresql+psycopg2': fn.Random, + 'postgresql+psycopg2': fn.Random, } class CallableProxy(Proxy): @@ -137,6 +137,7 @@ class BaseModel(ReadSlaveModel): class User(BaseModel): + uuid = CharField(default=uuid_generator) username = CharField(unique=True, index=True) password_hash = CharField(null=True) email = CharField(unique=True, index=True, @@ -212,7 +213,7 @@ class FederatedLogin(BaseModel): user = QuayUserField(allows_robots=True, index=True) service = ForeignKeyField(LoginService, index=True) service_ident = CharField() - metadata_json = TextField(default='{}') + metadata_json = TextField(default='{}') class Meta: database = db @@ -250,7 +251,7 @@ class Repository(BaseModel): # Therefore, we define our own deletion order here and use the dependency system to verify it. ordered_dependencies = [RepositoryAuthorizedEmail, RepositoryTag, Image, LogEntry, RepositoryBuild, RepositoryBuildTrigger, RepositoryNotification, - RepositoryPermission, AccessToken] + RepositoryPermission, AccessToken] for query, fk in self.dependencies(search_nullable=True): model = fk.model_class @@ -457,7 +458,7 @@ class LogEntry(BaseModel): kind = ForeignKeyField(LogEntryKind, index=True) account = QuayUserField(index=True, related_name='account') performer = QuayUserField(allows_robots=True, index=True, null=True, - related_name='performer') + related_name='performer') repository = ForeignKeyField(Repository, index=True, null=True) datetime = DateTimeField(default=datetime.now, index=True) ip = CharField(null=True) @@ -537,7 +538,7 @@ class RepositoryAuthorizedEmail(BaseModel): # create a unique index on email and repository (('email', 'repository'), True), ) - + all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py new file mode 100644 index 000000000..19b79df5e --- /dev/null +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -0,0 +1,26 @@ +"""add uuid field to user + +Revision ID: 17f11e265e13 +Revises: 204abf14783d +Create Date: 2014-11-11 14:32:54.866188 + +""" + +# revision identifiers, used by Alembic. +revision = '17f11e265e13' +down_revision = '204abf14783d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('user', sa.Column('uuid', sa.String(length=255), nullable=False)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'uuid') + ### end Alembic commands ### diff --git a/data/model/legacy.py b/data/model/legacy.py index c06df82de..46a81bf67 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -132,7 +132,7 @@ def create_user(username, password, email, auto_verify=False): created = _create_user(username, email) created.password_hash = hash_password(password) - created.verified = auto_verify + created.verified = auto_verify created.save() return created @@ -194,7 +194,7 @@ def create_organization(name, email, creating_user): return new_org except InvalidUsernameException: msg = ('Invalid organization name: %s Organization names must consist ' + - 'solely of lower case letters, numbers, and underscores. ' + + 'solely of lower case letters, numbers, and underscores. ' + '[a-z0-9_]') % name raise InvalidOrganizationException(msg) @@ -380,7 +380,7 @@ def remove_team(org_name, team_name, removed_by_username): def add_or_invite_to_team(inviter, team, user=None, email=None, requires_invite=True): # If the user is a member of the organization, then we simply add the # user directly to the team. Otherwise, an invite is created for the user/email. - # We return None if the user was directly added and the invite object if the user was invited. + # We return None if the user was directly added and the invite object if the user was invited. if user and requires_invite: orgname = team.organization.username @@ -390,7 +390,7 @@ def add_or_invite_to_team(inviter, team, user=None, email=None, requires_invite= if not user.username.startswith(orgname + '+'): raise InvalidTeamMemberException('Cannot add the specified robot to this team, ' + 'as it is not a member of the organization') - else: + else: Org = User.alias() found = User.select(User.username) found = found.where(User.username == user.username).join(TeamMember).join(Team) @@ -525,7 +525,7 @@ def confirm_user_email(code): code = EmailConfirmation.get(EmailConfirmation.code == code, EmailConfirmation.email_confirm == True) except EmailConfirmation.DoesNotExist: - raise DataModelException('Invalid email confirmation code.') + raise DataModelException('Invalid email confirmation code.') user = code.user user.verified = True @@ -534,11 +534,11 @@ def confirm_user_email(code): new_email = code.new_email if new_email: if find_user_by_email(new_email): - raise DataModelException('E-mail address already used.') - + raise DataModelException('E-mail address already used.') + old_email = user.email user.email = new_email - + user.save() code.delete_instance() @@ -614,13 +614,20 @@ def get_namespace_by_user_id(namespace_user_db_id): raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id) +def get_user_by_uuid(user_uuid): + try: + return User.get(User.uuid == user_uuid, User.organization == False) + except User.DoesNotExist: + return None + + def get_user_or_org_by_customer_id(customer_id): try: return User.get(User.stripe_id == customer_id) except User.DoesNotExist: return None -def get_matching_teams(team_prefix, organization): +def get_matching_teams(team_prefix, organization): query = Team.select().where(Team.name ** (team_prefix + '%'), Team.organization == organization) return query.limit(10) @@ -628,13 +635,13 @@ def get_matching_teams(team_prefix, organization): def get_matching_users(username_prefix, robot_namespace=None, organization=None): - direct_user_query = (User.username ** (username_prefix + '%') & + direct_user_query = (User.username ** (username_prefix + '%') & (User.organization == False) & (User.robot == False)) if robot_namespace: robot_prefix = format_robot_username(robot_namespace, username_prefix) direct_user_query = (direct_user_query | - (User.username ** (robot_prefix + '%') & + (User.username ** (robot_prefix + '%') & (User.robot == True))) query = (User @@ -1198,7 +1205,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe translations[old_id] = image_in_repo.id return translations[old_id] - # Select all the ancestor Docker IDs in a single query. + # Select all the ancestor Docker IDs in a single query. old_ids = [int(id_str) for id_str in old_ancestry.split('/')[1:-1]] query = Image.select(Image.id, Image.docker_image_id).where(Image.id << old_ids) old_images = {i.id: i.docker_image_id for i in query} @@ -1592,7 +1599,7 @@ def garbage_collect_storage(storage_id_whitelist): storage_id_whitelist, (ImageStorage, ImageStoragePlacement, ImageStorageLocation)) - + paths_to_remove = placements_query_to_paths_set(placements_to_remove.clone()) # Remove the placements for orphaned storages @@ -1607,7 +1614,7 @@ def garbage_collect_storage(storage_id_whitelist): orphaned_storages = list(orphaned_storage_query(ImageStorage.select(ImageStorage.id), storage_id_whitelist, (ImageStorage.id,))) - if len(orphaned_storages) > 0: + if len(orphaned_storages) > 0: (ImageStorage .delete() .where(ImageStorage.id << orphaned_storages) @@ -1967,7 +1974,7 @@ def create_repository_build(repo, access_token, job_config_obj, dockerfile_id, def get_pull_robot_name(trigger): if not trigger.pull_robot: return None - + return trigger.pull_robot.username @@ -2146,14 +2153,14 @@ def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=F AdminTeamMember.team)) .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id)) - .where((Notification.target == user) | + .where((Notification.target == user) | ((AdminUser.id == user) & (TeamRole.name == 'admin'))) .order_by(Notification.created) .desc()) if not include_dismissed: query = query.switch(Notification).where(Notification.dismissed == False) - + if kind_name: query = (query .switch(Notification) @@ -2278,7 +2285,7 @@ def confirm_email_authorization_for_repo(code): .where(RepositoryAuthorizedEmail.code == code) .get()) except RepositoryAuthorizedEmail.DoesNotExist: - raise DataModelException('Invalid confirmation code.') + raise DataModelException('Invalid confirmation code.') found.confirmed = True found.save() @@ -2310,7 +2317,7 @@ def lookup_team_invite(code, user=None): raise DataModelException('Invalid confirmation code.') if user and found.user != user: - raise DataModelException('Invalid confirmation code.') + raise DataModelException('Invalid confirmation code.') return found @@ -2330,7 +2337,7 @@ def confirm_team_invite(code, user): # If the invite is for a specific user, we have to confirm that here. if found.user is not None and found.user != user: - message = """This invite is intended for user "%s". + message = """This invite is intended for user "%s". Please login to that account and try again.""" % found.user.username raise DataModelException(message) diff --git a/endpoints/common.py b/endpoints/common.py index a21c6c8ca..12093b288 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -85,23 +85,19 @@ def param_required(param_name): @login_manager.user_loader -def load_user(user_db_id): - logger.debug('User loader loading deferred user id: %s' % user_db_id) - try: - user_db_id_int = int(user_db_id) - return _LoginWrappedDBUser(user_db_id_int) - except ValueError: - return None +def load_user(user_uuid): + logger.debug('User loader loading deferred user with uuid: %s' % user_uuid) + return _LoginWrappedDBUser(user_uuid) class _LoginWrappedDBUser(UserMixin): - def __init__(self, user_db_id, db_user=None): - self._db_id = user_db_id + def __init__(self, user_uuid, db_user=None): + self._uuid = user_uuid self._db_user = db_user def db_user(self): if not self._db_user: - self._db_user = model.get_user_by_id(self._db_id) + self._db_user = model.get_user_by_uuid(self._uuid) return self._db_user def is_authenticated(self): @@ -111,13 +107,13 @@ class _LoginWrappedDBUser(UserMixin): return self.db_user().verified def get_id(self): - return unicode(self._db_id) + return unicode(self._uuid) def common_login(db_user): - if login_user(_LoginWrappedDBUser(db_user.id, db_user)): - logger.debug('Successfully signed in as: %s' % db_user.username) - new_identity = QuayDeferredPermissionUser(db_user.id, 'user_db_id', {scopes.DIRECT_LOGIN}) + if login_user(_LoginWrappedDBUser(db_user.uuid, db_user)): + logger.debug('Successfully signed in as: %s (%s)' % (db_user.username, db_user.uuid)) + new_identity = QuayDeferredPermissionUser(db_user.uuid, 'user_uuid', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) session['login_time'] = datetime.datetime.now() return True @@ -279,4 +275,4 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, spawn_notification(repository, 'build_queued', event_data, subpage='build?current=%s' % build_request.uuid, pathargs=['build', build_request.uuid]) - return build_request \ No newline at end of file + return build_request diff --git a/test/data/test.db b/test/data/test.db index b58ef5c9eddf67a075194a2a5ebb8cb111ca9f74..c3c2c33bc3f7c682b6794a3c7824f6cedac6af68 100644 GIT binary patch delta 25581 zcmeIa2Y405`Uku-b573gNkZtMCxk!};NB=iyjsDJ|ExiWwV z*sc`;(Hq4A7VL{IqJd!HR*Z?V_dOKdmW#hzf>*~9Gb>@K#EadsP9$(FGtY$2P&8dx=( z%-n$v(fXYqhuuIM@vaA(LXqCxq(OQ|6G6)7Be`u3lGQViEUQPdcp8!glab8vB59b2 zq-Fw=$z@2qV~|u7A}Py7GBy)Qekziz5lB*EB;C6q z>D&=XL^~u}EfO^TSv1$KUChv?HRa6YQNn*AWKA8`2<3;dvu2xD6Xc8(om2Qc)V z9Sj>;VC%@zF!rdx9>vrgWP4DXbSDcuT2KLu;@ z)+1_7U~{|SftBI+G|%;tVv;Dss{?x&sqo$Z46i_k=qN*?s1=Ik2LwSfn{5`a-{IAH z+;*?d;x;*SPQTHu^BCcSdm(wI3zB9T$p6sD)2kwR0?DI(`VN~;vQZJ}96g{g z39K*c&N?uO5&ARzmY$@?=m+!weT_a(pQ1bHgY+)Cp01(GX%n48r_)O6qGhy*X47Qq zq(+)R%lp#qv;&nWAwQFE$w_jId_WG6*U0naDYAn+NbVx*$r`epG?6)EI;kWsQbvkM zHc2KJ?5D6shXW?7nq;O35KpqQzJ3n|PhRG;$%BNt0utN}bkn=RaEJBRX zM`;sGBHC%1vfF7QeRi^#@wY2Ue}3~+;78I_(uH)q1lHTc&T_pJ7%4hQI|V8Dos)5W zt40;rB7V+asU)c(?NdjmCyx!R2}$72DWo=}L+f=Ee`^Z4>FW1oG|J>?9*Gv%1<>8I zAR?c$Pe64KvNzewAThhx;~+N=u)9HV*RosL3bqvVb~dYHRm{&OvT`<-<+Ci7!iF&$ zGq8aymi1xXSVtDdLYP2*q2JT5=?VHN{fPdPzD-}JFVW}dlk_qA2>mGztL56 z8C^{0)0woEPN80UJuRi9X)ev6NpuLc&_o(fV`wkhmA0o^sv^IWpU64#6**3hk`Ku{ z~dCnYhVyWE?3ZqevPVK^(+H29p6K z8pJ7PXY{@=Xd{UdfP?4RH|!+)3_H(zz{NM%UJ&1>*iN>U-4A@cgWb+n10$EP1#A{@ zaw_w&3Si|Jmd7%ImqVGA>4BO3SZ@{y+ze%EdJ)+99X&(81b!Z-@6!Fi&=={mv>7=1 zFujj%29^fs%0_w<5Op4%L2H1j9y)=R09kWrIvojgHB%jpqkU;l+J!~{V;T94{0N*q zO}-#UfVGFnKJp6i_G$72*#^wLhuldxaQ9|%18D^I){|*u67Y9C$aeuSIF$?s6*7XL z^vAg6U!ZGrFMjrjO1(4ogOg#wuUcPe@i}q9YAyDFV_5u*%G4>G9VFR|`6>Kq^ z3sjiG+(3lUY!uL72r~f*VptELfR<4pz&Uyf;Qx>w1n^&=y8!x!=-qS!y^XE_)aTMV zz}!vC0r62Z1@Jb}fpmNf?Lj*N*p&PNP@f{70?-G^>j3jE@)&@8H`xFXuON#7;5sq| z;4LSk0o)Wa1fU&AVn`3tk!T4Oei6pt{{}4eUhzdmU7q&K8tbNsP+Zy`Uc%jIS|`T^MJ_xsg|4|Z<9yJGN5E3pw$Bq`cQaLcv$$G zut4z3-)j5o62}Ci*H`6h@Kv~KYpW_fu7=8*>MjGC8|RbxOy=_*iwk1wY5BPwV07-G z>wtOrSaLgr+euF>)Z3U%m?Ru#>7X)i3Xfn*=^^~e637AI+W>NshJr>Ek)dQOSQJKdS-GJjdct^;XrV(YMvo6x2+Z3Qv&&MQdgKMuBx43G!Vmp{CWTu!m-pzZ+UM=Iz5i5->Oon}W-caZU;H8i@j%)L-7Wc&`x zT(GcJ$O?3bwhZ?G^2L1xFiIkR$sUi(;diUh!u8kymYHadow>_aT3M5oDNHyN5@)a`^~r_dCRw@P>`7-gTwVsPq= zKBq-zu~{uTx5e$$`5ZQ%-QhKRZFUFWwwiWr%y49n8veI#B&Z5$^)RICl<)f{(QMvi$xhW|bMY*NM(%if}=U9WuV6^CrMxF7BPMIub zThl-lIu+HV9w`NfYO*=pZkyYoGkE+yoyF*P>Kw&Rt6S&y8I5L>-RW^yUH&>>{bYHl zu_jln(G;6upWTq+pPQOeRcB8vn`3c|n`tx6Xc%SkXN}7<7pG26n^{s)Y%Z>;@}$j~ zGfD4gC~KI`hb^Qrjcs~lH8`wJbJGBZ9(8Y04{zy_(dhQO{4SHu>UUXn78Aq_H{=VQ z)!{YyEk>)+Wp&ifaD%M0^k%F#Ezwjqdi0p`>YD1*xebLzL)IK$!K`xInEZOn*z}xQ zXMv$^#w^p!)GBFq?wHDk>Pb^g{P~5nZ`&ycMW{l3SA6HqfH_bMbSL+LNON>=Mx4FSs@7JeJHK&c?dm5=e zp#^?eK~Kn{`r~SuOoa${Luv=FzLQ>HxezmRm33uggFr9#tt3$(72n=MI;~wrV!BUQ zV(G7m?9zqYn4?UZj92o%j{mZP4B@NRt9mz^ZwBu}^g%+g{T7l$4Dt)Ii1AXDnvY!x zmdK>MnBTV&8|dYaM9Ujs7&8mt_VSn+?9dVL#t*Z4knfFPfmVVjfP9))(}O+PUYV0V z$IgI<`~>pMJ8VA)#|f}bd(rAX#9C#@FYHU_u)C?5-M}WX6JV3yW$oy% zbhrFM%C{et)QzQShIFT`tSmJ%wX}2;76+i4<}6IkF_fi`8&#Z9oLW?38@af9_E_IsHE+CFW%7SirvJnx0AT9b{@~>A-A$8vvbzNK1#~5c z*+0SJzX^@W3yMnV6nhx0ni#)pKlO%!O#s{PXWi+WsC|a=H$S1F|1mV?Ey!US=)ju7 zJA|@-A1IA2P-y`Q-bP#Are^;sZ16JD@-oW)1TZ77COUrtnRyF>oHHbd?;*z=gluq~ zkQ?X_J-B_7IZ+ciK7lmVR8|WHy@toE0y|^j`K!q6mU1)7enM<~M>6ZpD^i%2f3zB` z%)(@5B6hiUIdO23%%-=O*QSb{yd=8;SRb%J^gb0o^9}75dhI(Gafcogz!)a5r|2;} zbAUExssR?l{?i4gYI)DU3*E*`Ypw;ltmQ@jDPU=aJTU3cVA={&OyPhc9HYE0YD zP|%)a%Cs?lEQ-apa)gjn+Nr?L3%`&}^e0FuKLFxiFtbU+A4y>&h*O4{p7%&)lfs86 zx{37YvB@d9>AZb9i{w+*tNJy+n##HnGK?QfV>g5hR~qp%X{;yDPiGpwDxJk%!P&JD znI&#!u zWV1^hlg8CU@eSJv(9NXgYPi9+&ZvfCY{e@vf11go6T-9*ZFE|hPb@bDytiO z^$iU*Q+?HKmc4-eT7S#5%qwIo-r z7!yL;cIq1@wXT*nsN1F}V=>zxu;}&;c@hS zGfRUGU{F9ch>Q-hGgSdqQe0E`2u)Q!Kvmc?e`&C_g$tbQCpwsUK4Q~_l0}a9^VFN1h<8Q82^&_Rt=m#UDT>fk~9=1+Z zK*k5lU#(M#eEvFBA-OIHkSXg_9r^d`R3&7BET{Kq_ODlsBJE`SW$dgnee5j0{wb3c za9yN7)_o(_otD@9ckDFUea!gji>e+Px?Ny%=yufH3udvMIxN=Uad}}{vFQwuYjhT` z(X4a(ZEl^>;cywePNT`|@$;kY)P}|qpLcLHB)a^XnbGo6ZSWR4^LnjbvrXsdVfUIK zv_d{~JKaW|$KrB&3}%bNXE5;xBh(qaAsJkLkHulM1ZkmcFk`=B#6il}Q=$NZ6snLU znkvv#x)WRadZAg^8R!tbg%x3KglxpxaPy)&R6Tj#MpZOldxt8KxPlc&?@$fk(RZi} zWTIS=&STfB;(~CBO6C{Gpb&61^S&*bG9*e0-T3Z=YDII<(Q_F}b}KyF=%YO0$UXR=8$Ie5sXDAO4sEF(izr*fuL*R6} zp*uhs`xLOm?)3XTk#?uiq!}lCg7m%rP7y77#a^N62sLmjhFA5w1$J{iM5JEZQ;_itA9=4TJ!k(zhZPEy4m zd`G>QObb$$C@HCg$I$dv+I=3$VmbVIlM_T214jm!D3eg+0RZ zK!@n5eE9`+r&#b7F1yKMf+oQWaGEU^9nb{u_dA^iqsM0Q+YEfq1@*94lhtGKo1hbM z*#Uiv)8^5+%|5%%@36Z4W^gACyOl@(rq;#!3=V_UZ2~I;Hb!T$o2)vw$>-A<4PFzz zX7Rf{Ha`6~wKdk~1+qDuHl5LHfu6+zWsA$8P#cfYVf6c<M^*g;@86|ZF zr_TZQ$l}tuEGQY>E+Z_{HiO$~;qkw#V`5DPkIe;i*LjU@=tf-7-MC%Q<2j*-lb43u zWb*Lp-_^se1D9m6yG$|}mJQryEC9E`fV4@}@HjUe?ULh&S5-6U6a!wgq7 z^?U5lxHvH&hecKqVWqjO(C#_RCa2kN^!a=)BM-T#PKt&8&~J6i{G8JwuPz^$Iy;`h zdreR%Ivh@;i%+?zj_K#|`|W-|df@Pw(9-Gl>RcGH&S|u}d_K&Y%g?u6R6`qSwL4u7 zgF)wYL1AYxKndhtL2K|f#! zzPhlFOzT709s!EkR*PI4nSW#%HC$Lqs{%Tx|XPr zVcpqy0nQ@tLDs(mFCEYR0aNGwK;T9+9M1T7eaJ(7*;HA_`!#d}AAuYvYvka`)WI>S4PaYeyDE8KYU-EepPKV zYS$4mBRBztAE`t6){n5f@B9$cGP6x>++lU%0C_4R*b~Z@@&~=fep#u+_P{A-7kaju z@h1vG3i~oJ1=xPj_84pq@4;&KJeKO4(0d=pn4V@&pzR7}YKr@^2jsT@!t9H-i-W}E zk(!XQXxaBsk`u(KL)ugKCDhBv)atUbBEP)pchau2a|Op znS1GxZGU_`Z*6;xDkg%hlWF1i(7_#trtW`3X(FAdyhGIVr^1c=c~O(zN4Ax;czXQ6euam|L>dlbb*1o|uue$D z4EB#76Dso!0!bDoN!qK~AgdwK>VT+j;RP2%y6{C8LW~_+IjGC`mi-(r+Qj*Jyx;puCo}4GJ{N4&!ue2{ZiKO20_^pzB=icW z1}uRMEd=tpC{1Qh_Pl^~@-m1;ka)=6cVA&G{FA+brgt&^yk3*mlRX_IE>D9&I*rAD z1RCx`SS_!x=h5m8@E6~x)O^oKO?prEl+1VpohWu=AqAZ(o|1cmrt5=L<=FyFYKtdY zxgeAayGS4UAnVKm!ni<(=n>%y>VyVOhxTNVrg`0v1m?~*vH0~Nrq#k#w zBuz9GmIv?LD{1;`#9B>czkZazEopjFVMX)TlEzCkmD4qmI@Feh%BXyEkdmwk)eLI4 zpiUDxvL9{gr-;k^w@^*@aF@s9tFKqIA;K;En@~-TW@a2zf7wiQC2oZEVaenw#{UxPg#PQ>+JAjp z`s3}!D?cv&@&1*!HCNoe3i(5>xP2AYtFLjp(LUtT&{S94zY4M8O4MdzuDX4tz;xAZ zdG{d@2JzSSx3dp6HNL(^6Y4J*>&(r!k1k7hSh7mT6((n8<)+w6QtN{chB z#^S<^oXpHDr*TYHUPgXiS$TFwT57(vc(gOM&@NsDG>an;RC;@&yKhyhImaL>fr*C3& z=V6fW+&v|q{CXM>^L#{0Ar4_{FYYp>bKwGrZB+3KpS zak;hCWVf2!Gb*dRx@y-n#aWRFW4L99Y7yv_X*FoE`GjekT`;D zxYRq0g_FRS7=~^S;KNN@&5?cWkIAo1&7}|WsS2T)q`-HOB}sc;-D z{XR7I)e6wyX-;o06cASNH=)aJt$0z z$HRN!DETnk!e+wTstL~c*{p<>$Zy6^%_4|?@K8378#juM<_9*46n0QScr%)%^QH}A zk5HRgpM;7x2D1rrn}kM1^!eZOOl^dR;{T>+>Lu6IqqK)i=R#yHQ$YBdN8{k1~GJPv&4Da zlTsER_OaMa{><0BFkhP`mp zCC+0>K#N!)LGk@9K#^iA9S`m-v6TfeiiQ>A$`^{$lA0QNPDszj?j;@%r(i~ z`M9Z)lRO}=_FjB*G`1XNXMJs|G?qLR6lS156Wb7lh5ut~EgxPb-9#P^F7A)3q_g8> zCNmC!AJ7G#h7;2v3}+WYh?c|ge=S6tdLYvTczdTa2Rh$}bpnijh0p#AaM&uKNmzF= zSaB-2ydS~VaUWLda=8D>#)L;nv#JlaY7N^W;I&&&^*6@1=%s$$oyD1jDaPcS+ zX>p!q5D5{8RVPu!yQh~muiRc#T$*JcYc9#k&o#(hr0hKbGU|JH%Nxwvnd!(MUqU-E zopkL-+t@M{uXp}SfdsQ9^oh_S;*C!+zPCaesbyrJK=#o(HVBow3EwwO$|8?knrkh` zj79LYYN?TI4e|p@#p!B^lSlv9Y;29xIS4W|>cmgfNZn*+$kUCyEWqhK320qUZz7)| zE}{!!B=*r0jKPt92%KleLs{8`2!tKvWysqn;Qs&bxh^+P+6Yp{kK94Lja^`Nfoxkd zqq?EOGuc&*?ZM-rJ=xGd*4Db}>u1%}d0AIxmj3WWZI!FK6B~5-k(%n6zPbigH)bVJ zFjvSf1r=>1=#G)h9K117zEkU~n^sw04^QrPGUOCavcg;And+4X#O{*YM;8#1nYOnksE#0^7)(KEdM6H${*QCx|w9;>q3bPGAG;={d6H>8U9XRrJvE?v56V6 zz~``=5T`*9F7gyKu(CyEKjHtlgCs=A9cl3)M@Pc>ulQnk80$@@)y(u&pel?XxRZ2? zm*=*oZc;_dC}ags*Vek$=TR5QPu@v}^Xfa%xuQEs@2)a%<>%^s4Hfc36?HXLz9_!+ zP7wn*}_FZHezL4o1#MD5^lcivhKH7oDSJ`YBV`v7c_Yt zI)?*+3~qd3%%P9 ztNaX^0jTs>`n)ZZRO>7$&Kg%xYa2DCEIV)P*y>vA^lV@1G@q#~x6(Rh%-BlrjND1n zr?sEvm~JRE&K;XwGJ59p>Kf}+y)qkZ2AiQNI^;^wdH$`)nwDPq$}@7ziS8;xMusUh z*>B28&h|QdH3io929A{cVc2#&`!Q5nfUWs9nXKs#33CoGQw2WHAa^z*C zLD(9DyDKe1Xp}`bBFK`}r2k3)yVx4Q&Nnqm<$dD2rZzjG2CWbF!YQzFu;5m*7B;ja zf)l;y7kF~^CQ%bUxl%fE+u*?UbIQemPcE7`bbpViz@9mIqWwz%W1yqmd)RpMYcZli8M~4!d^EXQWVEsek!2=no zl&R=euu$RTIZ{O;h25y`D~#%<0r?>jyn30K+PrI-_&3&9=DET_ukd$|a0-#Dv&kMR z(n7k99)+2umT}mghc&OV#F@N2r<);*!%Kf4OWlM4qp?qyz=48;zGok!wu2|&lh-jA=u05v)tu zUVAV9P%cgHrj6x=@mh6Y;j#pNODAnC98>xvXvgu!!Pdl=6IeoP&ZyTuXZwvNpt^oXeQ#eohKx0&ku;3TS=_`7rz1%Nh zt}u5^a!!#OFIzcE;Qh^ykY@qxx$68OtHZN3unsTG<8=;6lD-(iC+0hfC($q>Fk zrFBhmq%w~!g=;;sa&*myC~*7vu*@`c(s%#TB6?4XZ<|paNx08ZV^kePls-u zG+%~87}mz0E)T4#OAF-pkEAut6^Er%W|S9%C^W!Y$n?-<1?Bgf<;~o!bJ+Y?PzFEz zyF6Yy+{jEeJE9fgoaN<}%Q3vTw+w^qyHacTbIUPfgFf}m9bOyv5TCF@nkgoj+CSlZ zWWnqF@Cr=XfF(;Rt*46ueDTegvc%h#h{@)qBY6BRn6d%KQ@Yq@tZwEjZjtg5qD~AZ zgCDMaD>zGA{wESyyIU&cvv*6&_>)J(ES~qCsA+C|Nou0XctsEl)&$&|Mb40k2qk<< zaACaJ;y9x+M{#`f3u1EfBU`1hwE59RB5fr*f7~#IJUj-YFhuLtLt-pum(S>cudvr^ zgPWVxVAr{PeuvIvvRk}vn+H*y9)A1=3{Y44xHNLUek9Mj5tBCJS;LJb?>{_o-Gn)FG~BwoK;qN6YZ>NjWctsErMD;iookj$ z<&6U#UwO=?|M?wxmWGjd*?%b$I^Z$T1y?{PJQeh27Ief*petSto$(#e9p4Wf@=oZI zcSEOaf@b*;bj&BocjPx{o;yHK*&oiaPMV67C*lzZKMvvW)j0d$MraQ1!lwNKG-aRB zbFgG}hyEZm7uueiA+-K8g7A}E+|sCG5t1Gu%#lQIG@11C6no0tmTQB4PVl~LWKk=( z5cq&{M-($)k<2cq;*Q#prkXE zbhd_dLLc=s*f+gNk3%qe$~7jvS+5G#8Vq`0OUb04)KaqOD_cr7{nVC{LqAO^H5!cu zeNC{^Xwpw_DOvROEhU?NMoY<|pCz&`_@G1IDfnfF-sseKZc~nIQ|{TOtZ!4c$a)(2 zC3x0kp!zO6d^0>H%=&Kpz$Q`G#jfwwfyG^FVb_}+`rdr?Cb0_+{xHiHtj?+^D6!;H zBs5YZHsNX6*oEufsu*Zu&;J1|C*RZGU`y!+yNV8H zv*g3-G7sx>9aOlRz|1062r?dhO``c8l9=G+JFc2)FQ#_Imw@LdTBe_u8#S zugz?=c#O07@I>t*J~0j~#j~ptn!iZY_9TJvH|Ybeg)#gtht|MHchSbc!}kpbcoOOW zJGHf|Ha2Riype_q;{;g$GU<8fvU9+9Y{%&h65Ky!zb;4fQx8fHwekqp5?grGVtF4M z^6nQ&Pwoxpdl!Su8$an&lK%FsM|s>5ka=-)-{WB&E~N1JORzU4yj{9u$cip6@t#XT z=h>YP3{gj%Udq=m1)Xn~(NxswLjJux=>}<2g87|~Z_m25a>`%a|8)Np3%KQ%ronH_$c*ZMY^4g=Ksz<1hj9@{QOg~1<)j)C>`M zOzO2`Y}oTJ@f8;^dqb!6ST`oI|4@G90%k8h(l^Z!{k(}U{7ssfpqmxdH1h9zAD8DV zTqqUrcWpUP78&i$XJ3~dXa(#w`PDmrz$fC4Mp*5KvKH`RKVl&9Q_krew?9?Kcl`*U zYk!?FAY!6+GI#w1pbs5-=j_pEA|BwHpV4jA*F7^rj-UILSN@D{ht!$INg)}t`Ex%@ ze1tRWljjRf2O8v#{;OrpZh$lslp7Rj25#7MWtk=@Iq)?$EWOW=1io02(BEY=!LJ|T z69%xBa-a^vXeWT^1F&?<2Xzh5D9D#9w7D8HZ?x-n0i%-<^4hVU(WAt98H z*XHtf%C#9h=>=R~DAT^L+-Sy)>xsgChY2{K=24udMPTXo)6QU~>Pf6{5DFIAd*#Ys z^Psz>{S1mh+0E`cfnB$=L-c6Y*wi-+N+SDZ{iWS+mCpt->uf%gd{7EvP+TT2R99|? z2hk)l7jAXWm{VPaI1JyQY99<%bF_ z^&Zyow)4E__dwU7@s-u159-W({r5oE-q$^2Oz1i76P|P)=&GxE(>8MB(j0#LJkWK- zgYNAmPo2G$XZ-+SVc2`N>SA54S%&>z(5D=I3&O&Qx1{&E<0U8?p4%^_@G4nsh|bb- z4zGkP-khbqm22Ri5Im4bz+po>aDti^hk}*BkiQOOXdk3goI}~U`EadN1F)H%#+Ox2Rx492Aw`QCF1@uHN>Hsof-ocO4Seadc4{AJ|JaCqAs(u)@HRRmbbNu(-+RdW) zNk&ZA6t|w=*GIc)uvE7~AMX7A0T~;VG1Sqn$T~8b3SOn@7I9>+&V#fkY9XJOy_5=*Jkn=FN;YrLE3TxC$e2rTlb_;ifzfp zYcsS3{5^Q0G_TIqP9}VJ1~7^Pt(t!vrR@$gs)DAu0+?GL>ja0PDjJQW#+Cx9P9V%c zIekJLH69Xt$b zr9t3>0`)f}@Slfi@8h4_z&%ZbhUn$t+WbQKGqL0=0WLV_DZyEWG4N2b(+r$;=z<5+ ze0Vdh0lV}dh~G;%Eb%mq|9!x^Prztu;XJh*O#TDVt-gW)rw_pB%aMy4Nu$D|+N}B% zg+)cdX49uCENZZ9v+L6o7WLAl&7n_Mp0XSCnaWcr*p2!u<*8uVZqjEfPhGmSoAslD zPg(W3!KbWxyKSmIFZh^zp|{)h`N2j`ePOVXQ|~b7i-K*Gvcsq!t+av2F;;04ls$@- zHo>yPsvoC({?etxrZ3@1>s$Eh1M9$7m&trJN|A`d{$`!nN0v_d4pumNr(UaN-HG0* zhu|1`x!6v&a0hFRMtvA>+AQ{B7JUTI-YiBHD7|t*&IuME`6%6M5r^88-AY%3^-hQ0 zBV*pR)yMpqNGv2NJk1gI$l!eL>4wE=eW6oHPRf@@xzrtt)7oq1fKlT^-{ModV{w{h z;(dP!8N{o81Gyk}Uc1z>(*Q>jK-_tK6m6=Wv zaEM|polO_vD9u};6yHK0z`1)*(milXc$>Zt56%*yp=Sn zRT7b?Mn0;M(qPWuJStzSs7lGzRITJ8G(^cYRHNh~6_s3~l9FqwR>?zYsFH`#FeMMC z;Y!|)wo~#58lmLvX?snh{GkrCgHq9vc2x3Cw3Cu|rk$0%3+xs z^6s>|lJ}rJl)NYHspP$AFD36ydn@JOgp8HK-jfA4DF-#+MA&HB4K!4w0#1WH3m?Fv z4S3wm`Brx!+t&LV{Rr+e9HYK-HyOKMoL&=@gspLcG5G5ZWT$-!oc{ESS^J%`47tjJFFQkP^UPOzOd^8=c$;)ZEl8>k3mHaw-osv(W6O{aVdcBfY&EG)h9t;9{`-B~+R zqwDafV`R5Id_!mL7ExXL+#=2V;m3JO7wk(R_SqkZk4LQKC%Rx?ie9<*pe~R#fRE~m zeMxv+|Fm!+@NlOgEHTk_(bFQ7o%N*bRxsVz5XtYT@DoZQ!S>XDb zAq)IW_=$9FQV)3pl*kG}HiD8Bt!7a2wh|RxOO#XeWL7absJE=qXp<47J@O?nhntT` z`+~IxR*A_xalZ&z^#w7Lzqe0>g%2<8Kr)4(36IpurbV*8#kkmF#v_@sff4ZPO=Om^ zu{n8!Hk8m_5wn9v@R9I9I1 zKBDTM+5AF#Z2o=5#D6%jp~}nWb^t@sXUaPT18-k_7w_8%hmha@0P!MDOv8tVWVua6>z2>q_f+G<4r9bq`44wxJi7)Tj!UrCu`rC>?^@5c&XBZTFKVn-s)CY zRaf<6oA6^NB7$03+mRWIT6^n1%(S_;r<+i#`k&IPkh@KK5^71Z*HCTU%X@L| z>FIskrD<`fg?ndc2X}h9;Ql-VIw_rQxt_thruIm=!{wz%A zo)c<8O6YW=dr~)lq_~ZmbJuqZcBgj@L#%g13%ee57j=Hcy)JN5{d{x;Nz0uf5pHv* z2~n_IvubHb$Bu_&h!c|jxsaIpLgL>862I0wqzGS=;A{0Q2VJJ?g9e>wlDM_pAon$< zg%tlP!S0sl&fHgDujPi*?q5}y!|NsZDn5x1;$67DCHH(2B;GcFX_$LGF68hA3Elu( zcL?tWz!on&6mUc90*p6H@CAGV{}X?Rci_zr8(v+O;NI*v0OKtZ==vl4Hol0r;3E$M zh5oU7OUI0Ok(Q+R2S_Ea+@>XNy6P*HUipTytzx%g2>TY>l5xv_k*}2ZkrC-n(kl8e zJ%jE^6EB5e%MrY0ntsq53)DJ6Ll zL-0_kly%pHhmtKfs6@8&?_>)_g8R^^>kZ6r;z?svL} zlBbVRqvd_t>GNh!c25r~W~2UjV?xX#cVBlyP&%txtg35aX`yo)g9+PWA^OxW>iHn| z;ou1@-$RqxYAj~m*&!?0PJ;#ohQ0CFukPPNU=?0&w=_B9hcRTuQ7WSb_H)fdNLCd= zvaA;*O9LoK5k3s5=8HI$%qiQ^}(owS<7%%nrn5VDYh{IFeg z#NyBC2okds5{ny>Ekhxx`ICwu0u8y56hQm|nMS>i`X=ykPam$%(RSLgd9pEM@2M&203)Mz)atL&J89~!j6Ab1j6JQ$u7tFY!D`1N9Vj9{F zK-I*s3qZGsGr0VY*nN8di100fKLCW2h2oNK?g062Bv72nFz+I80N2F}cM-BEgiJ9) zogmg(Zb47&F#rU50V>37fHzzII|Z}5Ny*4KadcZm$dNax2=|7_u2L_=mFptJPFhx9 z6TXUa-w0}hz-D|~}M{pvKr|=mG{u!TXhBv0V&uBU` z_^br~g3mSsljc5a@UIb(P30ma_zyUR&jZ{23V8m<_+7jWZ^p0U)p!}M!*g*ho{Y!h zGHl01I2)(n7;M8j9D#$eKW>Q?7zuv}SA_GzN#QHupzyKquCPtmEW9eL7M2Nh!d#(N zm@JGH$^^SmBxDOILX2P&bV7ts6D;@(Ed_;u_&@k7{CWN){}q3b|CoQ5-^Op|U*%Wx z%lJBeE?>(}=Ew47yqz!Nv-uQ0hPUxLK7tSC{rQ%>f=ApR+!gLTcar;xJIH;^y~}Om zHgm6XtGQ)d9XFS&-^I3KzrKI2{+!WNqMVq)FCm^d`?&FVbOjjZ$10 zK1hQ91PSFQIJ>^X-vXaFg!khQ;IP|{-++VfA9xLZ9xuU*@I3rDo`EOfDm)sy@GxAA z2jC2xjAL=fP-wg|5a zFAJ-Ln&rZ?!UEw*VWu!mm>^UNBZX37h)^ix2x&s15H0i+j6#$UE`$i3gmyv;K`BW2 zoBZ$mRsIM541a?En*S&NDgP0_hku)Yli$Rz<6q=g@Jsn;_^0_f{A2tSejGoBAHh5L zL3}=+#rNmq`M$iB*YiF2ZhTOal@WyqIB=`$gZg|=)$pAqaL``Emw{YP<8Sa$Aeeo4 zF9-_Ta6Mj+Ujd?d4ll+Ffox{s>3AX#&M54}LxFU1aXL-{;^~D=SOetK1qWb1ARrZ% z3AccReikkXXMu>02}gv_fQ;S~-Vr=NNUsSm2`hn=+`{}C;R&Fosls@n0_dqk7%UV3 zMWqS}LO-A>gP<0|gkYhg&{n`eRy6+?{~Hk2MgDuf0Z8jGe}MlGh-(MGmEQ>DwU&Q@ zCqQ6N@w53FAhBw`oF5KEHjvNbGl9(F_&&S^2rZKD3iB5K>e~&v&_u|Ab*Umh4M(#2 z(3s=YlHEvJGnMNu!Ph~2x&U&~*FZ3z;5|StTktv{mgV>vAeEVT3J}Uj>;N*!!To_q zdSX40NC@r#1fs+gkjGWw0uaa7!XY4yJ;HV%jCH~qVY%>(Fb`;Aickd<;Sh>}9{LNh zKn;4KJJ3Q0p{5m(0>xhkLb$-60x~$ne*#3Xo!*Pn~!dUAT0x41xU2VHDjAT2kTCLgV)*V8V_RsVH-2f7_Xh@vQ&RDl%KzZO>8 zC=f02uO2(zUOhtm9?_!(*IbNR+();!rPWc-6w3bZD~Cz7ttpv=l2I$^?(`SZ$+B?S zXYzAQKJzm>Q}KbaR3%d#L``>YB5hV)p71ui=MhyZ>sWYRBxarYSzSI{k-Z| zZmGDJQL3#Hd=3cyO(66-{1~nT0v~|m;bRYna{>uxgnhy$fxy@65Ypjm?J6kwbNuK0 z>->}O-DLp{b^yBjntP9XjeC}x&bffD)LdKD4b``*_f%V(Y^}EaOqyiwqc(e7?^7%2 zUjFEk1YM$*(xd2~q(h{qW!bV*@?p$O<_ETjJ)#(>_)ytK`LyyBHzB;vj!=;M-I{*xdC(#la_!w2%ZgA4 zVpnK6OH?cL#d_Ww-}{*Q-Tj;4;%w)g{7)LF~vm*8Nj|bbBpHv`0TO9_rZFC==s8`{CN!!dbL|hl~iDsbU z+$S~$1&Q2-Q^GMj8bpQ){wzO&yU2}ET~kd`g)0A{?B1lYmg_~~X&^LPB(*#%VHGSJ zL+VD+ZObz9&G{MW{qwYjocz4FLUTq+dXY9IJF6(J*qCFs=A{?u@)JzjtPF!T)ne84 zH|J$0#OqUI(sE7t>G^TFdacf=*6GwbOORgQ(`4&uv_{zsR+CLXMa4@Dce`_E?`U0< zqV1oPX4a?0TjGovdFj@o{`qmbgxvly`k2h*!q@_Bs;My7Y|b&}<>eW)g_%Xh_=5g1 zMoW5OV(b8IM)pH{)7$h~Q;w38be3pX9ZZdlag3ilX0*L(xDyV|$iEg7%#`A!60J3{f01r@j#)n?Gt)MD?3C&ZYmvh|DnB-LOocvr+z9)aQPoi; z73EC^UemZv<|v!RrZrk8vb@CqZtqo2d#SUkvFE&)icI5#qU5r&iP~`q<0cmCCKgYK z8m@O$y7b8@lPe~aXO@+XjZfAmXKQVVF-e-#q@t)X6}pG_?CZG3XpS;l%_ggUD&w0s z5QI4I@na@9tIAwu&eF!-3&-{^o;=(%AvdKc)^4kaG8N{VMvu4lAF0V0k(^bQotd9f zVa!e+?kt_GnP?u7o0U@#RW_!=^r+sA=9(yj&0;WGrpoVG^(y8V#TgSb#u#m-y28w;iloubF%`x~ z_a3EdqqQ3K=IJtCVrxR38+Wv;$u7)J&nixi*Jc)`WG1BN$K;se5-sssYg$^aSzoLz z&M3?<=O-rQre<3T4TTwrDY;f#Zk{2@W`OS_x2VY`-`QiGR&O&JYGqPh(z#aFuaU%D zRnF=WP3Br!8KuwCX4`Ud3dbkf#+OaXi_6r-kIyp~#zh^Uk(HQN9T%5AK{rC5 z9G{{aUp1=4nVURrcu8LD#Ps;MjMU*{$2h91t96zbb5y!fW2tnyCu|A6`_-7DOnNhX zaT7swRo`8~+aFnE>A0#2N5wd;L1#7Orsc-w$D~`~)6FPI%84&ZF{dW!;!}(J>l0IQ zVh1E9q{sI+$Dn?djj&c=DD9B9Acg7bxCb(kl1Iou2=Nl)N4mT#da;A)$ zGCsSoD9*jRUUQd+nWFRtt;J*%k9ZCnB%ryaV*}U`8bIxR4OhcS`8?<^^P8|(uaVh% z>=||(>WKG>D)Uu*3ADp+!JP0<{26fcbD#z7y}wQ4DbkK?I*O$3k9YO%C_sm6z*b-Y z%ff6t9G?RX`3M$-8^W&p8$*XwO@g$RNB=1!=e$_{9R|%c<3H}}| z_s8%Tpz7}j`~4pLGAQ{M@N4(A^8}5izM$PN?Hi^N_DjGT;KKgG{`-LHwH!~o_kXIY z`A_$4>hx5afg7e;zNL!(pKo8yy$38{qZ+Zv=>K#Z0U2sUM$ygeV$Z+L5T;HSzeXuO zf7|ln(e0)_M-SfKjHKK_eWdP7N82m^KJc;jD&eFAtTz$DNj&z!-3p<};6)VkA8(g5 zNd{Ym{*9gTZdA=3QHX9t^q+1Ll_(BK(cJ#rw>{mPXV|{ofi-M2qTauTjh|Dj`{e1+ ze^Vf->9P;Eikn12cFzJ9JR z(%(i& z$aXucazAkcyN}$6Bz;(>S;7OxcP$r5yU+aEHax8`wJ5cJVPa8+%~WWL&DIvBnhH{i z<4ps!1#vmL_=4=>+|+zivU}HWUERsQWrpWv6c~#v@d<^osRJ_mCzz6KIf-$Z`Q`zc zshP1kNhz@zMG3a-q*!PW!->>J{Y%^atw2V+2L?-RGt^itvI_DCjW`)z1E=o8!0NIT zBL*S3v{NX8+xo@uyspGyHCfGOwboEtsy3FGOVw73(WQ18TxOTkX>l0MrB%+#ik|k; z^0G1GoYmvTRg7|uxg*h7gS5JyMsrWSs4AIE2Awz=guj&FFCW%FsOkOmVRWA2@AT_K z^}7WB-VCatzxvzMuJ+)v?PIp}+})R7*HH zS1zmr!+N2lqAsRg`e%wWKWB9^gKc-WOF7ktNj_ktWtgXv`pYPX<|N;bQ*@b zpmb04FhnDiA(pi#gX~DG^5%Ox5KH#kQ5M|5idW@-mEiQ)JC1vq&6BXC2E`5U{o6%2A9EXa5>F79XU~jv^D7!V|oOE29;4UAxPwO zx6MJ8Ahj0$LV5&+jH?)0Y9Hqe=@}IAAhf}2Jh}}%{SLFV08(F7l-^>{gZ3vy3P~PP z)XHL5v%yqq);jEJR|liZ34SUi7PZ{~&MKBti^*a(mpV&JtmM^dlo)Ui9-Yx<^PM1~ zmk_>j{{l41D&EtfwStn;NTKq<|0eFzZ_U6D^6GHpMh#Lj_&o{Se7)BUWH^~0Lh-%t z8I(9VDu9krJbdnV$Gi~^3cRry%x#8>SL%Of-zf1+xx0eRFQ8~>q;a_fbl4TR{DFS6 zHqoT~DWs|?)jAD^l2VJ>;L=;wMkn|S*-Q0$wZ-H#x^za1MW@%(l!js@C{F<@i_%+PsG{e+~(Bz66wNYR%)NlC81Jm-|_YUy#{%Zua z@PdkNdd*SG0PpVm(^6W3yd={&j?0Y-OEW#>czgJHEo!B}0DAcm(n^lV1C->CR}km@ z!mmI23x5rDkc%47HQb!4G=?JQHzGfSxWla|B%u)XDs7;5OXtX>vXSy*OdfNbou)J> z-&f_Se&^=$8A5AeFI-ZTN=UyKP;XT1W0WgiKt{T@>j1KT0oax}6@0f4n74GbkGx3*JCGV~N` zM}9nwS`q6{s23XL8}tiOs)gtIpU_;?$f)4xxzG&l7s0txk2?Bub0v5aT)@kPPlYi; zINXqr<%780+}!&p6$H??$=o+kYd8*Ur6#@AuGSe$C2$-#?P`a<)UI|qTqakE)@*WU z^frP+81?X_y3FcQhs&lnm00ZhQY{JEiqxS7oztK*>Rf7_ zUI(s*E~g8afWe}+I7}9k&Tg|?oq95MD>8*zbk>p*tH}jbB7f1@MRs9>Hl?Lbm$B6B z)SF%8ldY&{sKwwcaat{AwaE^0xzS>@s2whyUhOP#n5`v7lhtXpk?=QBNT}KBEOj~z zcD3FNK*mzLO>NZ!Z!sHmX0y>{aFsaiWXzkW_aKwL#A>kHO=_p!0?ew^Zdcp2Mw7bK zVlb3~JEYN7YBoz5i6R(D@`l6ZgMG}n@%Awk=nDE$m&2j9S+sVw(FhFDYBd_wdaKJ; zVuC?C^mZcmpy*J$$)ztbJ1lCO-L8W*vuf22V2WxTOw#JI8?10@k&zx0(#33dSS%LU z0=rFbR2xk$gW75^n}98vZ6zhz5{F4kHhPdc7L2eqm!$;ywTjcz!n#^r&_r*s1DMX> zvYEZwhYIhOz~$9#mzE|zKKIeWZC8TUWcMzV;4j)aeXh%*VQeox11L`5&+#YCG;8lW zAwPZ(_4daDC3rq~&U^@-GT*^9;+x=L^a_w;%>!+-7SLqihiF|e9wlCk{Ro!0ui-HI z0GwmDfPdBs>;`v|8axhl_! zvh^^^3dSHR`I?@D`_!Xw!T&CNJp18Z@C*!O2X1;XFNzPIAT9L`O|~3Gxxx4!V%tYJ zP(9FX)?uI0@%k}T6ePMA*L#=w1k8FTcwBuCmxEt}huRJN9+;L%@OM=22i1}Fw7A$Z zQs02mm%NXpAs8WgQ8MkqlQTAC&Fx;>=WmuJag6=^oPGlS(R* zTwo&^VlIS5W@V)p1IyeM>{5p#9*0yojYFS4oAo)IR_P*mE9I zd{If|w$wcU4TFt)Nby<~WqU~RrNgveR0}#)LZ{NlKq%=ddzX2e?aDr@kSm;uk5oCT zx54M?Wt=CxjFVB8WU1uYx|r7NtVtsozkmQlc1bBeR0(b^?L0q9DR303?KFb%OShrg zenfK>JW=-l48BAQe@5-l7&1&sbrlPY%3A+2#xE71dikKDrBooQ5CICA2T>D1&5*8F zK^__HLvnKxND!8vK?vA*6;#o3U+J-{AQ0R=$5&oOEyx!qL9qN^7?U{Ny@(k;it#hR zI&7CyA*hPfcB1^qCvp%XZpo=YrhGKxXMx&Kh6+X1zFIp&^ndp08R_v=^=*r`~b}J3%u`PjX~CGMc!>q#dpLzd*z-dO3?Lf+{} z#RZ8cB`3`Hs;~G(xPf$mhfg?GfGg%c@XLM3rRPq{tIon5u}@dO!$P<~ItD$U&fck3 zJ-S-?=xT^OxEFbi*HCZ=J9~fqz17S;)emq#`90MSuszvBs%vaL?{qG`r~UyxNI$yz zUNgW0twnVW;bju|ujyZX;Gk1YymGz7NS1|BDwj4b-5PI<(Wdo}PwcNV>eI|cndyeO z^Z{AUhn{xtRpm8V4=9s zzjLmWofaxq`M1aPC~G@Jq(~Hh2UDsz^haU(TXy?gN}=GRCAbY}e9`wWf|i?wNOT@` z;Et#u`g7?F2Ah@y=2OS+E!OJvCZa5$jz!%5@}@QuP>nxl^1Ojps=4 z9K7)%7aUn1(`3$2YDyoGs0CTHL>4XkQNE1n#r&*%TcuU)=JL5S;23)Yn9v%&1wU74 zCA@*wNv=p5Bz?ez$NMKGZ>*yFqG!Z|qoh@>tX9GJ#f4F|k!52_os+7Zq-r~rjTZaZ z{_*Wp7Ws4=m4u%4Y1vTFf+TE*=o&*I6@_X=jXJZ#Jb`LSK2D%?O!&k*hfNHcaX(zAMCfT^8OV9wV`ot!)lSiQ65KXI2XVd* z_9enGT62FxoJ2i=9($lPf$GIMONWY{L_^C;Ln7{@nPqB(c+auQVrtzZ^*^hA@=on;_bbV-1@wHryLp(mSE z)sCbx&|J^wBPlH+?#JN-o<}^7Qv=acr1fmd)@T6#S9UV-cmv~C2?fTltBuOo{q);t zxXtq&z`s^g5SH^VVXszDt^Wp?=$}J;DCdb5*m)1J!Dbzx$-Ifw6c+_IO7OQDPTXIG z>4F9VFXrevmd!S87C)przcu77Dya+I<4WZ*ae}&LA@{DUkKTUgTrF0GLDL+BE zX-~#+>L0Wi{`XuVB+DbkFL5gKv8V$c1%)zB)j?UH?5Jo^Y*jqPe#CBLXR+0637gBt zu?Dsa+nS}BpPAFlVX!iu$KSA%*)s4WPho9r1nbW#m_L~F%va3E5Va8nZtZ(jOI3Yw zXUs#)#zEnP@TTyhuuzyN)C>~(3IPz?Za_b}8Fv*3xpvsUQXI zhcsZ0)=ZVr7-^<7{?=+Hdw)+NGsEds$pRgWG&YmQX4crOni3V=@%B$glf0d!+j`4~ zdD{;4{WNHXcz+HKq7z6kp*s-mcv?!f^?;+l18ozF{G`Qxv_Z_ZsVQnhtC7ZP&{&Nc zt4U)uYpfPcVM{vf-ogU!knVuxdxzosndj{n@&mm6=6WG5HjP!QvFbEAUP#~1Y%e6_ zv%HX*09k`H7L&$e)>tf>3?HcXXSxs6q)GFFrg{ft(P}I@jYY4q7&QHT-P|ro@wN5- zO!l?aYLdKd6TNNCCXLywF{%J7R7rT`yk@HjUgZFZ5%^3)KI}R_=S

    Fw>YX<;vKk9SIXdOP&}w0S#( zyw%&G#S5vkYIHV@Nv|>A`_trY=&Lot9wO34L$^uv4x+KtqA^-EMw>?O9YoVYT@zeJ zi$>c7mqv8LBQZ0m4kSxW!znS1QjpK1XeG&xqT@)*C_0+%MNhp8;Q?)kF*Vc=^TO3I413 z---VF+rfYN|JTufU2fX~FTs-yC&4N32r#dY!It(8go11a7W5Kc2|=T7ux~yAjA$wz zk1HT>v;+^v1rR!#iW6`@2p%trp6K;Sf(cP{ zNckD_gsLt3y=pbPk}jw6R0Fwc#b45Xim`&9aGrXOKS-UFZ&l8f-{7{PK3Ich$yy3) z`2^VuoLU)2VJ1c%A^A;qn!AO^GY6#m_@`87+1Aq8=wn%d!bATduka4bm$-XyQ8KSD zE2)X0W5~^EbSgQbqmxOAo{lH<0xCx8{mL?X(J^v^4-$BGV`*6s9V63u0p5iy(h|R` zR4Soc(ut(zJQYKto&bCux5H0e8yZ4}NK_3J>{8uEFt#nU3L~GVK^i3Y?Uew@E5?3K_4W898fk)4p>3XTu3F7&##G97pR!q6H>t7d?Vh~`A+uo zoxyiLNyW(9`T$lc?*Oa>3}QEOqW^_TAdXX1w7g{_)ETJy+)k%@^iNYW5ZP#_yAdoF zp4$$Csrepq+YK;*s2%Y0r|&4M0+xBDudz62F(iA1SesZQz9~TA5#lzy1aZM5fF^l< zukeCuwyF;|n0tvgac#Lv{B+d;dN6f>{fq6YT+gH{ud!R%ayUV+DUQ&W6bV!xYPNj5 z(kXvOas&lP_sa5=0cbrVlYK9Hfmtk_&U_;^A*YO2%%@+Ht0nWr?!13>{ROf#md+qi zD!PtnI>U}<^`fJRCV*ziD=hr{P(d#uAD*X@Nwpj1HR(K+Nsc`OIjw@uBxeHwq(Dr> z%_Nnbp)5><+Z_No*hO!SA+JQ!NhDth%W+-_3u6g@7H2O2Qdo{;Vs`-Ci9nhm1!CE* zi-4f=B6I}Xm`rLEB6`?5a*hS0uXKW+uu+g)pi;!$i57w7rB#Y)#K?I+t!iXTY`S5$O7cnV-O{5cL3ZmTsWPbr}2vo34 zFZkN2$d#w*XqmWI&l4@oU`ZM+%?ab-dgLnH4IG7gfp;K^c@=Q5S#Yf~93phnaWwF& zP~cQ@;87PKI{8xw)!rbi5avT{_Gn=cL}d3CdO*;z5`?{Tz-B(+@4pkN*K(VXXX|1r zi^|4+jo0G;tqAJ-RRJ)hiFYV6(&;(sd^Yy?vFrzIY2wulD5kICwfF^yXsg3dfyJu^ zBK@mzIZWIR7gKpS6C}wv+y`4A-ais|1u@tE!S34lu51EUQm6C9qccp9ig)i9Apqbo z^spTsbD0a#@k_v_Rt>W0VDQaKz*fK)h+D&=-hjaNJwiS3@o3=SA;7z(5D4A?!t_hP z^yfmn=mV}7TJ9F|NMD9qdj_u(;}BQfKPpils>_m*Rj*N9{1f{pC1$4fFGwpkWmyZ7 zVj(P6f=p@!B|67<%E;DcrJLi^b*bjutdzJSaS9YZDgkQv;bCaOveVlnN<8nRl@-YddwVl(}ACwo0qzYZ!kLIS!-9eYc$M){2@U)8|X zLQHEFj=bAW93@GcO@*VU$@tk+8JbTTW>eG90#WsEOJ>ccT9br1pp`#6hZ=+ydalf& zFd{8hQwCCNrTj_d8cIrrt)?8P&Zp2qLCYql;uWi?0Y1CK%gF7!if^%&`iUIaMkU{G zUzpLv0`W3h(!}_Xw*!W8+x{Ft&h4Pu$BOol0C<@PuMmD0vLR^S$Tj@W9`%7J>dOE3 zNYDQt9_cAg3m#D#9<{d!!o(z!K*>M}$dD-@1O3U*fL))*g{x*Nk0}Q$b}Gsg3~Oh$ zGri?6$dhGfWYN-P(vI{rI*i&25x|oq*Chk%V*E(&M0lGi_`(#8X6nRanprCVhM@qL#MOa z5iJkJz5o3#vZN2->{qiss_^@()x^{naJKQA&36jA1aBPm1)TkgBEKu#r0hqE`vFe6 zF5?^N&DGD5>7jr#nA!SlRG*tB(hv$b^=cLS8&BM!Ff#S~m5E5ZO}tCqhI|FryaGOs4n55o-obp1PqCw8-BhQB!Vr)Mctzg>8l{OJ!k1D@L*5>fr~60)`f;8bW< zw&0E@v?D1U0jF+R@cM*`8&}C|9Ra7`+Ueg1w{Q0fiRJ)j&&v0wW*=2w#f@%%OA zdu_Gezgm%kT_A3%CI<;aPav zqL8Ci8^9pbTKTSWw6cw2n_?h)on69ezz*~*6Dz+YUo20NU6=hsmMgs>-6$Of^W8?5 z(}C1}Y9idw9Y^z!N%D>NIk6y5!36qMp>fZ?v<&hc7)62Dtug)CvIZ~|_sD)C#**TAy-{z&9^l;9cs9krI0d8Hy>{*Gkj z6S+LA&QZf?GTKcgdseYDWP>0}R&}BmHBLY)kxr8Cma+f6;Y+oDdjbm2q!{{1iahxY z461Md{Vh@v`zDZjca5bPFjtCVs|N(wXo~z3Ks>9W#VLr;9)SJ_O?rzBS5?v{$VX%8 zXz!FXbeKfiPdXbO-71&0loiNcke!xC$SdSq<=2^LW)A%av!89n7P2eY?-bn?m2|!0 zO~s#JqkB^Msj7{tShY%ZhU)<@qrAo4gy(eT@t?!fJcHobmUB3Y4g+y=`@LmiN#mZl z!#O=s|3rO7YR^;sJWpMrhQPyI+d$TqUIBf`{wpNA7J^8956pPVr_l|S!(=_bQ0w60 zKMu0zex=$YuO@X{f69Z>n=NRHZY&U0Cr`WWbbI8Td8D+jMEZ&JaajvlzU+B;JR@8_ zM*fCx*7wf&;WL)@eb}7E`8|9!?wz^rwrVG;A5jMS5J_G@C3<=!(`JeoQ^AaQWE&_P zBZKH@vTX;Q>{hD&``9ZO&Cd5vUG0q)rHpNn?1|FzUcexCx|u_t4< zWcPW}pRAY)+|lsj#uLeDzuL&W(ZF!qez}SZT>Sbsq+v8LTutG+)Y#-7%gBs!z-fN0 z_o1}(;H%`%a=_WQeEstUuA*^d?HIr*M2-Bh_1BB*NlFFa>{av2i@9lUe@|Yk0GulF zWFH!#kN->>3IS(mn}y5tvosZCa1r3NwDIql-ly7;vHi)MY`}^8n0{`XxA_w3kpnn6#m)9DzWyYbEXVRBN<{X9#m z$mtxw+5O{@ulG9h(L6FU7jSmIsn756N#Z18jTh(MA?ZN=74jBohzFd#YR&50j2ac0 zod7u7tk~GQ)3@PIkh2MNdVY%^Lcj3aEEyv1bQdW&udl%|!Z+}gR(JjwSbUd4_rxfl<$+*$ThOlGPf*A>T`bw$$GnI z(g*aD$WyopScxa?Y5EPCO&`KQP&jz3{s5Y=XX-fmIBKM`!J<%;D!c>_bUg-7ls(OL zS3R!$MmZGTryQkFuqDjfOka4PvcK${ELQrQGyvYGjG#WGg3%PopOV3KG5#caH!O_$ z$Sp^7c%g~x3ANy z*p{nb`m6oMj*H2h^?wDBD2y&T4E+9SJ?8ha#=99s@+ z(eA`VT@b^^lcMM7RgB-k7yJi2b&S{^1C+`YZDhhXnLSCvV}P<-Ks;|6)T0%NsRfj6 zLzk}%SoTspSyBrqWvgPoWljt-6VpsUsk-pP&K4U}D@pxK7-q+7R~y2*1|1~Dv*;CV zI<@GsCM;#&v>^k1GYo_GEaBP3P_lj!-CeFZcp>)c2GTHzPOoYCk7={ozqKJ-R4IGP zJrXvXox-k_pJ4YhU$EiqHRf?epdyKzqNq?TW%P=76&IAPmA#quN~dxGqkwy$6DoyD zuNtU&T(w^Hh3Xa;4$s%E<@R&e;2EPNzLL2G59ICPFUmba8?b=7goVsNct!3cJR)X* zH)nEZgNx!(eDwdlItiQes8=VuM!*I|SRkZA4{f$Lhy!?2J>h3-s8A=x3gv;}BCV1Vk;yrlxajfs%$JgmlR7pDj1$999@Xq8? z=)dwW@VA1OIA0VWr|eEX`isgYu{Wt4GUj)xi}*}rl=zn90P@ByDy{K#NO(7M>pprV zBI&h&aTsw?^bPl8;o;&FmKpAK@J{44BmJ0Xg^7;yo#WB6-4eL2TqA#z-N)QySoRl1 zfFe<(E|Invow7Cay1Z1`8?21IlnZXtmwf(R3S(w@X>1C&mJPd0XYvLgo!zD~*?E!5 zKxb?gSdJ7Mm}1#(<_JFOvq5ON6hxBVp=-!o@tMq#LmVSo0{j#o#3XtPttC(U(;+Bad@0knQsNV) zBw#Rom~80)Pmy*b3y09xy>FkoYa_doYm4Yti9e>hkPCJ6W8$NvpOQ-P=Xy7tD?VH5 z!y>+L>bdX^-4zk*E;@%~a&(uRF5 z8l9OrlDp#T!$mpnEoQ$czOt?E6;>H|NbI(oxR%SNNsq_rqh#m+QDmzDk?lB$Y_|n1 zkU_~jJN+e*+UZ0O1SYLOdA8V?r%#pLtZbQ-B{3o_d3IkcNBnhkQE;R$*^ zk93D^$=vmodh*&nVEX;?I!!2CH&FHe0YO}zApigX From faeb3b9a10e8fcaec96b435bd79cc126df1c368d Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 12 Nov 2014 17:02:46 -0500 Subject: [PATCH 09/15] Update tests to use uuid in session --- test/test_api_security.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_api_security.py b/test/test_api_security.py index 5d4674f9e..97ec3950d 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -79,7 +79,7 @@ class ApiTestCase(unittest.TestCase): with client.session_transaction() as sess: if auth_username: loaded = model.get_user(auth_username) - sess['user_id'] = loaded.id + sess['user_id'] = loaded.uuid sess['login_time'] = datetime.datetime.now() sess[CSRF_TOKEN_KEY] = CSRF_TOKEN @@ -93,7 +93,7 @@ class ApiTestCase(unittest.TestCase): final_url = self.url if method != 'GET' and method != 'HEAD': final_url = self._add_csrf(self.url) - + open_kwargs.update({ 'data': json.dumps(request_body), 'content_type': 'application/json', @@ -1069,7 +1069,7 @@ class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase): class TestBuildTriggerFieldValuesSwo1PublicPublicrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="public/publicrepo", + self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="public/publicrepo", field_name="test_field") def test_post_anonymous(self): @@ -1088,7 +1088,7 @@ class TestBuildTriggerFieldValuesSwo1PublicPublicrepo(ApiTestCase): class TestBuildTriggerFieldValuesSwo1DevtableShared(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="devtable/shared", + self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="devtable/shared", field_name="test_field") def test_post_anonymous(self): @@ -1107,7 +1107,7 @@ class TestBuildTriggerFieldValuesSwo1DevtableShared(ApiTestCase): class TestBuildTriggerFieldValuesSwo1BuynlargeOrgrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="buynlarge/orgrepo", + self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="buynlarge/orgrepo", field_name="test_field") def test_post_anonymous(self): From 10b627c2ad83fc706d4376297fb41cb79adf818d Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 12 Nov 2014 17:03:10 -0500 Subject: [PATCH 10/15] Add user uuid backfill --- util/backfill_user_uuids.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 util/backfill_user_uuids.py diff --git a/util/backfill_user_uuids.py b/util/backfill_user_uuids.py new file mode 100644 index 000000000..cfc0dcf64 --- /dev/null +++ b/util/backfill_user_uuids.py @@ -0,0 +1,48 @@ +import logging +import uuid + +from data.database import User, configure +from app import app + +logger = logging.getLogger(__name__) + +def backfill_user_uuids(): + logger.setLevel(logging.DEBUG) + logger.debug('User UUID Backfill: Began execution') + + # Make sure we have a reference to the current DB. + configure(app.config) + logger.debug('User UUID Backfill: Database configured') + + # Check to see if any users are missing uuids. + has_missing_uuids = bool(list(User + .select() + .where(User.uuid >> None) + .limit(1))) + if not has_missing_uuids: + logger.debug('User UUID Backfill: No migration needed') + return + + logger.debug('User UUID Backfill: Starting migration') + while True: + batch_users = list(User + .select() + .where(User.uuid >> None) + .limit(100)) + + if len(batch_users) == 0: + # There are no users left to backfill. We're done! + logging.debug('User UUID Backfill: Backfill completed') + return + + logging.debug('User UUID Backfill: Found %s records to update' % len(batch_users)) + for user in batch_users: + user.uuid = str(uuid.uuid4()) + user.save() + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + logging.getLogger('boto').setLevel(logging.CRITICAL) + logging.getLogger('peewee').setLevel(logging.CRITICAL) + + backfill_user_uuids() From 606ad21bec8b81e96c42f33820c94f5884953cdc Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 18 Nov 2014 16:25:11 -0500 Subject: [PATCH 11/15] Apply reviewed changes. Adds a length to the UUID field, renames QuayDeferredPermissionUser parameter id->uuid, adds transactions to backfill script. --- auth/permissions.py | 4 +- data/database.py | 2 +- .../17f11e265e13_add_uuid_field_to_user.py | 2 +- util/backfill_user_uuids.py | 46 +++++++++++-------- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/auth/permissions.py b/auth/permissions.py index eee8d75ff..ae398092d 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -58,8 +58,8 @@ SCOPE_MAX_USER_ROLES.update({ class QuayDeferredPermissionUser(Identity): - def __init__(self, id, auth_type, scopes): - super(QuayDeferredPermissionUser, self).__init__(id, auth_type) + def __init__(self, uuid, auth_type, scopes): + super(QuayDeferredPermissionUser, self).__init__(uuid, auth_type) self._permissions_loaded = False self._scope_set = scopes diff --git a/data/database.py b/data/database.py index 0ccdbf54f..95c0331d7 100644 --- a/data/database.py +++ b/data/database.py @@ -137,7 +137,7 @@ class BaseModel(ReadSlaveModel): class User(BaseModel): - uuid = CharField(default=uuid_generator) + uuid = CharField(default=uuid_generator, max_length=36) username = CharField(unique=True, index=True) password_hash = CharField(null=True) email = CharField(unique=True, index=True, diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py index 19b79df5e..7c193e054 100644 --- a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.add_column('user', sa.Column('uuid', sa.String(length=255), nullable=False)) + op.add_column('user', sa.Column('uuid', sa.String(length=36), nullable=False)) ### end Alembic commands ### diff --git a/util/backfill_user_uuids.py b/util/backfill_user_uuids.py index cfc0dcf64..701004a23 100644 --- a/util/backfill_user_uuids.py +++ b/util/backfill_user_uuids.py @@ -1,44 +1,50 @@ import logging import uuid -from data.database import User, configure +from data.database import User, configure, db from app import app -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) def backfill_user_uuids(): - logger.setLevel(logging.DEBUG) - logger.debug('User UUID Backfill: Began execution') + """ Generates UUIDs for any Users without them. """ + LOGGER.setLevel(logging.DEBUG) + LOGGER.debug('User UUID Backfill: Began execution') # Make sure we have a reference to the current DB. configure(app.config) - logger.debug('User UUID Backfill: Database configured') + LOGGER.debug('User UUID Backfill: Database configured') # Check to see if any users are missing uuids. has_missing_uuids = bool(list(User - .select() - .where(User.uuid >> None) - .limit(1))) + .select(User.id) + .where(User.uuid >> None) + .limit(1))) if not has_missing_uuids: - logger.debug('User UUID Backfill: No migration needed') + LOGGER.debug('User UUID Backfill: No migration needed') return - logger.debug('User UUID Backfill: Starting migration') + LOGGER.debug('User UUID Backfill: Starting migration') while True: - batch_users = list(User - .select() - .where(User.uuid >> None) - .limit(100)) + batch_user_ids = list(User + .select(User.id) + .where(User.uuid >> None) + .limit(100)) - if len(batch_users) == 0: + if len(batch_user_ids) == 0: # There are no users left to backfill. We're done! - logging.debug('User UUID Backfill: Backfill completed') + LOGGER.debug('User UUID Backfill: Backfill completed') return - logging.debug('User UUID Backfill: Found %s records to update' % len(batch_users)) - for user in batch_users: - user.uuid = str(uuid.uuid4()) - user.save() + LOGGER.debug('User UUID Backfill: Found %s records to update', len(batch_user_ids)) + for user_id in batch_user_ids: + with app.config['DB_TRANSACTION_FACTORY'](db): + try: + user = User.get(User.id == user_id) + user.uuid = str(uuid.uuid4()) + user.save() + except User.DoesNotExist: + pass if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) From 7c8a438b58fee83213de504ce17d1a12dae3769e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 19 Nov 2014 14:07:54 -0500 Subject: [PATCH 12/15] Rebase migration on top of master and add backfill. --- .../versions/17f11e265e13_add_uuid_field_to_user.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py index 7c193e054..7a9da4bea 100644 --- a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -1,26 +1,24 @@ """add uuid field to user Revision ID: 17f11e265e13 -Revises: 204abf14783d +Revises: 2fb36d4be80d Create Date: 2014-11-11 14:32:54.866188 """ # revision identifiers, used by Alembic. revision = '17f11e265e13' -down_revision = '204abf14783d' +down_revision = '2fb36d4be80d' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql +from util.backfill_user_uuids import backfill_user_uuids def upgrade(tables): - ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('uuid', sa.String(length=36), nullable=False)) - ### end Alembic commands ### + backfill_user_uuids() def downgrade(tables): - ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'uuid') - ### end Alembic commands ### From e863b96166eb56c8959576a7e04cb3909e3007f8 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 19 Nov 2014 15:32:30 -0500 Subject: [PATCH 13/15] Tweak the uuid backfill to leave the uuid column nullable. --- data/database.py | 2 +- .../17f11e265e13_add_uuid_field_to_user.py | 3 ++- util/backfill_user_uuids.py | 16 ++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data/database.py b/data/database.py index 95c0331d7..2cb1a51ca 100644 --- a/data/database.py +++ b/data/database.py @@ -137,7 +137,7 @@ class BaseModel(ReadSlaveModel): class User(BaseModel): - uuid = CharField(default=uuid_generator, max_length=36) + uuid = CharField(default=uuid_generator, max_length=36, null=True) username = CharField(unique=True, index=True) password_hash = CharField(null=True) email = CharField(unique=True, index=True, diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py index 7a9da4bea..56943ee8e 100644 --- a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -15,8 +15,9 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql from util.backfill_user_uuids import backfill_user_uuids + def upgrade(tables): - op.add_column('user', sa.Column('uuid', sa.String(length=36), nullable=False)) + op.add_column('user', sa.Column('uuid', sa.String(length=36), nullable=True)) backfill_user_uuids() diff --git a/util/backfill_user_uuids.py b/util/backfill_user_uuids.py index 701004a23..ab9ca4567 100644 --- a/util/backfill_user_uuids.py +++ b/util/backfill_user_uuids.py @@ -1,7 +1,7 @@ import logging import uuid -from data.database import User, configure, db +from data.database import User, db from app import app LOGGER = logging.getLogger(__name__) @@ -11,15 +11,14 @@ def backfill_user_uuids(): LOGGER.setLevel(logging.DEBUG) LOGGER.debug('User UUID Backfill: Began execution') - # Make sure we have a reference to the current DB. - configure(app.config) - LOGGER.debug('User UUID Backfill: Database configured') # Check to see if any users are missing uuids. - has_missing_uuids = bool(list(User - .select(User.id) - .where(User.uuid >> None) - .limit(1))) + has_missing_uuids = True + try: + User.select().where(User.uuid >> None).get() + except User.DoesNotExist: + has_missing_uuids = False + if not has_missing_uuids: LOGGER.debug('User UUID Backfill: No migration needed') return @@ -46,6 +45,7 @@ def backfill_user_uuids(): except User.DoesNotExist: pass + if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) logging.getLogger('boto').setLevel(logging.CRITICAL) From aa623957771df38f9af8282b46ab1bfe12f579ce Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 19 Nov 2014 15:43:28 -0500 Subject: [PATCH 14/15] Fix a test to use the new calling convention for get_build_trigger. --- test/test_api_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index d24d8ba59..435f8beb7 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -2342,7 +2342,7 @@ class TestBuildTriggers(ApiTestCase): self.assertEquals(True, activate_json['is_active']) # Make sure the trigger has a write token. - trigger = model.get_build_trigger(ADMIN_ACCESS_USER, 'simple', trigger.uuid) + trigger = model.get_build_trigger(trigger.uuid) self.assertNotEquals(None, trigger.write_token) self.assertEquals(True, py_json.loads(trigger.config)['active']) From ac0d0ce36a813d1e050c197e6e22d66c3e09c463 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 19 Nov 2014 15:58:47 -0500 Subject: [PATCH 15/15] Change the order of migrations to allow migration code to work with the updated model. --- .../versions/17f11e265e13_add_uuid_field_to_user.py | 4 ++-- .../2fb36d4be80d_translate_the_queue_names_to_reference_.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py index 56943ee8e..ba0d545a7 100644 --- a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -1,14 +1,14 @@ """add uuid field to user Revision ID: 17f11e265e13 -Revises: 2fb36d4be80d +Revises: 313d297811c4 Create Date: 2014-11-11 14:32:54.866188 """ # revision identifiers, used by Alembic. revision = '17f11e265e13' -down_revision = '2fb36d4be80d' +down_revision = '313d297811c4' from alembic import op import sqlalchemy as sa diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py index 76e698471..5d7e2f672 100644 --- a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -1,14 +1,14 @@ """Translate the queue names to reference namespace by id, remove the namespace column. Revision ID: 2430f55c41d5 -Revises: 313d297811c4 +Revises: 17f11e265e13 Create Date: 2014-09-30 17:31:33.308490 """ # revision identifiers, used by Alembic. revision = '2fb36d4be80d' -down_revision = '313d297811c4' +down_revision = '17f11e265e13' from alembic import op import sqlalchemy as sa