')
+@internal_only
+@show_if(features.MAILING)
+class TeamMemberInvite(ApiResource):
+ """ Resource for managing invites to jon a team. """
+ @require_user_admin
+ @nickname('acceptOrganizationTeamInvite')
+ def put(self, code):
+ """ Accepts an invite to join a team in an organization. """
+ # Accept the invite for the current user.
+ team = try_accept_invite(code, get_authenticated_user())
+ if not team:
+ raise NotFound()
+
+ orgname = team.organization.username
+ return {
+ 'org': orgname,
+ 'team': team.name
+ }
+
+ @nickname('declineOrganizationTeamInvite')
+ @require_user_admin
+ def delete(self, code):
+ """ Delete an existing member of a team. """
+ (team, inviter) = model.delete_team_invite(code, get_authenticated_user())
+
+ model.delete_matching_notifications(get_authenticated_user(), 'org_team_invite', code=code)
+
+ orgname = team.organization.username
+ log_action('org_team_member_invite_declined', orgname, {
+ 'member': get_authenticated_user().username,
+ 'team': team.name,
+ 'inviter': inviter.username
+ })
+
+ return 'Deleted', 204
diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py
index 4ec20bfdc..081641e00 100644
--- a/endpoints/api/trigger.py
+++ b/endpoints/api/trigger.py
@@ -14,7 +14,7 @@ from endpoints.api.build import (build_status_view, trigger_view, RepositoryBuil
from endpoints.common import start_build
from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactivationException,
TriggerActivationException, EmptyRepositoryException,
- RepositoryReadException)
+ RepositoryReadException, TriggerStartException)
from data import model
from auth.permissions import UserAdminPermission, AdministerOrganizationPermission, ReadRepositoryPermission
from util.names import parse_robot_username
@@ -205,7 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource):
'write')
try:
- repository_path = '%s/%s' % (trigger.repository.namespace,
+ repository_path = '%s/%s' % (trigger.repository.namespace_user.username,
trigger.repository.name)
path = url_for('webhooks.build_trigger_webhook',
repository=repository_path, trigger_uuid=trigger.uuid)
@@ -374,9 +374,24 @@ class BuildTriggerAnalyze(RepositoryParamResource):
@resource('/v1/repository//trigger//start')
class ActivateBuildTrigger(RepositoryParamResource):
""" Custom verb to manually activate a build trigger. """
+ schemas = {
+ 'RunParameters': {
+ 'id': 'RunParameters',
+ 'type': 'object',
+ 'description': 'Optional run parameters for activating the build trigger',
+ 'additional_properties': False,
+ 'properties': {
+ 'branch_name': {
+ 'type': 'string',
+ 'description': '(GitHub Only) If specified, the name of the GitHub branch to build.'
+ }
+ }
+ }
+ }
@require_repo_admin
@nickname('manuallyStartBuildTrigger')
+ @validate_json_request('RunParameters')
def post(self, namespace, repository, trigger_uuid):
""" Manually start a build from the specified trigger. """
try:
@@ -389,14 +404,18 @@ class ActivateBuildTrigger(RepositoryParamResource):
if not handler.is_active(config_dict):
raise InvalidRequest('Trigger is not active.')
- specs = handler.manual_start(trigger.auth_token, config_dict)
- dockerfile_id, tags, name, subdir = specs
+ try:
+ run_parameters = request.get_json()
+ specs = handler.manual_start(trigger.auth_token, config_dict, run_parameters=run_parameters)
+ dockerfile_id, tags, name, subdir = specs
- repo = model.get_repository(namespace, repository)
- pull_robot_name = model.get_pull_robot_name(trigger)
+ repo = model.get_repository(namespace, repository)
+ pull_robot_name = model.get_pull_robot_name(trigger)
- build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
- pull_robot_name=pull_robot_name)
+ build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
+ pull_robot_name=pull_robot_name)
+ except TriggerStartException as tse:
+ raise InvalidRequest(tse.message)
resp = build_status_view(build_request, True)
repo_string = '%s/%s' % (namespace, repository)
@@ -424,6 +443,36 @@ class TriggerBuildList(RepositoryParamResource):
}
+
+@resource('/v1/repository//trigger//fields/')
+@internal_only
+class BuildTriggerFieldValues(RepositoryParamResource):
+ """ Custom verb to fetch a values list for a particular field name. """
+ @require_repo_admin
+ @nickname('listTriggerFieldValues')
+ def get(self, namespace, repository, trigger_uuid, field_name):
+ """ List the field values for a custom run field. """
+ try:
+ trigger = model.get_build_trigger(namespace, repository, trigger_uuid)
+ except model.InvalidBuildTriggerException:
+ raise NotFound()
+
+ user_permission = UserAdminPermission(trigger.connected_user.username)
+ if user_permission.can():
+ trigger_handler = BuildTriggerBase.get_trigger_for_service(trigger.service.name)
+ values = trigger_handler.list_field_values(trigger.auth_token, json.loads(trigger.config),
+ field_name)
+
+ if values is None:
+ raise NotFound()
+
+ return {
+ 'values': values
+ }
+ else:
+ raise Unauthorized()
+
+
@resource('/v1/repository//trigger//sources')
@internal_only
class BuildTriggerSources(RepositoryParamResource):
diff --git a/endpoints/api/user.py b/endpoints/api/user.py
index 43a08508a..3fd59d86e 100644
--- a/endpoints/api/user.py
+++ b/endpoints/api/user.py
@@ -12,6 +12,8 @@ from endpoints.api import (ApiResource, nickname, resource, validate_json_reques
license_error, require_fresh_login)
from endpoints.api.subscribe import subscribe
from endpoints.common import common_login
+from endpoints.api.team import try_accept_invite
+
from data import model
from data.billing import get_plan
from auth.permissions import (AdministerOrganizationPermission, CreateRepositoryPermission,
@@ -19,7 +21,8 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository
from auth.auth_context import get_authenticated_user
from auth import scopes
from util.gravatar import compute_hash
-from util.useremails import (send_confirmation_email, send_recovery_email, send_change_email)
+from util.useremails import (send_confirmation_email, send_recovery_email, send_change_email, send_password_changed)
+from util.names import parse_single_urn
import features
@@ -117,6 +120,10 @@ class User(ApiResource):
'type': 'string',
'description': 'The user\'s email address',
},
+ 'invite_code': {
+ 'type': 'string',
+ 'description': 'The optional invite code'
+ }
}
},
'UpdateUser': {
@@ -166,6 +173,9 @@ class User(ApiResource):
log_action('account_change_password', user.username)
model.change_password(user, user_data['password'])
+ if features.MAILING:
+ send_password_changed(user.username, user.email)
+
if 'invoice_email' in user_data:
logger.debug('Changing invoice_email for user: %s', user.username)
model.change_invoice_email(user, user_data['invoice_email'])
@@ -176,22 +186,27 @@ class User(ApiResource):
# Email already used.
raise request_error(message='E-mail address already used')
- logger.debug('Sending email to change email address for user: %s',
- user.username)
- code = model.create_confirm_email_code(user, new_email=new_email)
- send_change_email(user.username, user_data['email'], code.code)
+ if features.MAILING:
+ logger.debug('Sending email to change email address for user: %s',
+ user.username)
+ code = model.create_confirm_email_code(user, new_email=new_email)
+ send_change_email(user.username, user_data['email'], code.code)
+ else:
+ model.update_email(user, new_email, auto_verify=not features.MAILING)
except model.InvalidPasswordException, ex:
raise request_error(exception=ex)
return user_view(user)
+ @show_if(features.USER_CREATION)
@nickname('createNewUser')
@internal_only
@validate_json_request('NewUser')
def post(self):
""" Create a new user. """
user_data = request.get_json()
+ invite_code = user_data.get('invite_code', '')
existing_user = model.get_user(user_data['username'])
if existing_user:
@@ -199,10 +214,29 @@ class User(ApiResource):
try:
new_user = model.create_user(user_data['username'], user_data['password'],
- user_data['email'])
- code = model.create_confirm_email_code(new_user)
- send_confirmation_email(new_user.username, new_user.email, code.code)
- return 'Created', 201
+ user_data['email'], auto_verify=not features.MAILING)
+
+ # Handle any invite codes.
+ parsed_invite = parse_single_urn(invite_code)
+ if parsed_invite is not None:
+ if parsed_invite[0] == 'teaminvite':
+ # Add the user to the team.
+ try:
+ try_accept_invite(invite_code, new_user)
+ except model.DataModelException:
+ pass
+
+
+ if features.MAILING:
+ code = model.create_confirm_email_code(new_user)
+ send_confirmation_email(new_user.username, new_user.email, code.code)
+ return {
+ 'awaiting_verification': True
+ }
+ else:
+ common_login(new_user)
+ return user_view(new_user)
+
except model.TooManyUsersException as ex:
raise license_error(exception=ex)
except model.DataModelException as ex:
@@ -422,6 +456,7 @@ class DetachExternal(ApiResource):
@resource("/v1/recovery")
+@show_if(features.MAILING)
@internal_only
class Recovery(ApiResource):
""" Resource for requesting a password recovery email. """
diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py
index 1cbd46192..637033ab6 100644
--- a/endpoints/callbacks.py
+++ b/endpoints/callbacks.py
@@ -26,7 +26,8 @@ def render_ologin_error(service_name,
error_message='Could not load user data. The token may have expired.'):
return render_page_template('ologinerror.html', service_name=service_name,
error_message=error_message,
- service_url=get_app_url())
+ service_url=get_app_url(),
+ user_creation=features.USER_CREATION)
def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False,
redirect_suffix=''):
@@ -85,7 +86,12 @@ def get_google_user(token):
def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
to_login = model.verify_federated_login(service_name.lower(), user_id)
if not to_login:
- # try to create the user
+ # See if we can create a new user.
+ if not features.USER_CREATION:
+ error_message = 'User creation is disabled. Please contact your administrator'
+ return render_ologin_error(service_name, error_message)
+
+ # Try to create the user
try:
valid = next(generate_valid_usernames(username))
to_login = model.create_federated_user(valid, email, service_name.lower(),
@@ -147,7 +153,7 @@ def github_oauth_callback():
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
user_data = get_github_user(token)
- if not user_data:
+ if not user_data or not 'login' in user_data:
return render_ologin_error('GitHub')
username = user_data['login']
diff --git a/endpoints/common.py b/endpoints/common.py
index 52715a1d1..37ae80ee8 100644
--- a/endpoints/common.py
+++ b/endpoints/common.py
@@ -82,20 +82,23 @@ def param_required(param_name):
@login_manager.user_loader
-def load_user(username):
- logger.debug('User loader loading deferred user: %s' % username)
- return _LoginWrappedDBUser(username)
+def load_user(user_db_id):
+ logger.debug('User loader loading deferred user id: %s' % user_db_id)
+ try:
+ user_db_id_int = int(user_db_id)
+ return _LoginWrappedDBUser(user_db_id_int)
+ except ValueError:
+ return None
class _LoginWrappedDBUser(UserMixin):
- def __init__(self, db_username, db_user=None):
-
- self._db_username = db_username
+ def __init__(self, user_db_id, db_user=None):
+ self._db_id = user_db_id
self._db_user = db_user
def db_user(self):
if not self._db_user:
- self._db_user = model.get_user(self._db_username)
+ self._db_user = model.get_user_by_id(self._db_id)
return self._db_user
def is_authenticated(self):
@@ -105,13 +108,13 @@ class _LoginWrappedDBUser(UserMixin):
return self.db_user().verified
def get_id(self):
- return unicode(self._db_username)
+ return unicode(self._db_id)
def common_login(db_user):
- if login_user(_LoginWrappedDBUser(db_user.username, db_user)):
+ if login_user(_LoginWrappedDBUser(db_user.id, db_user)):
logger.debug('Successfully signed in as: %s' % db_user.username)
- new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN})
+ new_identity = QuayDeferredPermissionUser(db_user.id, 'user_db_id', {scopes.DIRECT_LOGIN})
identity_changed.send(app, identity=new_identity)
session['login_time'] = datetime.datetime.now()
return True
@@ -202,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None, pull_robot_name=None):
host = urlparse.urlparse(request.url).netloc
- repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name)
+ repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
token = model.create_access_token(repository, 'write')
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
@@ -218,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
dockerfile_id, build_name,
trigger, pull_robot_name=pull_robot_name)
- dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({
+ dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
'build_uuid': build_request.uuid,
- 'namespace': repository.namespace,
+ 'namespace': repository.namespace_user.username,
'repository': repository.name,
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
}), retries_remaining=1)
@@ -228,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
# Add the build to the repo's log.
metadata = {
'repo': repository.name,
- 'namespace': repository.namespace,
+ 'namespace': repository.namespace_user.username,
'fileid': dockerfile_id,
'manual': manual,
}
@@ -238,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
metadata['config'] = json.loads(trigger.config)
metadata['service'] = trigger.service.name
- model.log_action('build_dockerfile', repository.namespace,
- ip=request.remote_addr, metadata=metadata,
- repository=repository)
+ model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
+ metadata=metadata, repository=repository)
# Add notifications for the build queue.
profile.debug('Adding notifications for repository')
diff --git a/endpoints/index.py b/endpoints/index.py
index 5c8d7058a..eb52971cf 100644
--- a/endpoints/index.py
+++ b/endpoints/index.py
@@ -19,6 +19,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
from util.http import abort
from endpoints.notificationhelper import spawn_notification
+import features
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
@@ -65,6 +66,9 @@ def generate_headers(role='read'):
@index.route('/users', methods=['POST'])
@index.route('/users/', methods=['POST'])
def create_user():
+ if not features.USER_CREATION:
+ abort(400, 'User creation is disabled. Please speak to your administrator.')
+
user_data = request.get_json()
if not 'username' in user_data:
abort(400, 'Missing username')
@@ -420,7 +424,7 @@ def put_repository_auth(namespace, repository):
def get_search():
def result_view(repo):
return {
- "name": repo.namespace + '/' + repo.name,
+ "name": repo.namespace_user.username + '/' + repo.name,
"description": repo.description
}
@@ -438,7 +442,7 @@ def get_search():
results = [result_view(repo) for repo in matching
if (repo.visibility.name == 'public' or
- ReadRepositoryPermission(repo.namespace, repo.name).can())]
+ ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
data = {
"query": query,
@@ -454,6 +458,7 @@ def get_search():
@index.route('/_ping')
@index.route('/_ping')
def ping():
+ # NOTE: any changes made here must also be reflected in the nginx config
response = make_response('true', 200)
response.headers['X-Docker-Registry-Version'] = '0.6.0'
response.headers['X-Docker-Registry-Standalone'] = '0'
diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py
index f3f4d6a77..4a195fbd7 100644
--- a/endpoints/notificationevent.py
+++ b/endpoints/notificationevent.py
@@ -1,8 +1,4 @@
import logging
-import io
-import os.path
-import tarfile
-import base64
from notificationhelper import build_event_data
diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py
index 773779fb7..6f80f83d0 100644
--- a/endpoints/notificationhelper.py
+++ b/endpoints/notificationhelper.py
@@ -4,7 +4,7 @@ from data import model
import json
def build_event_data(repo, extra_data={}, subpage=None):
- repo_string = '%s/%s' % (repo.namespace, repo.name)
+ repo_string = '%s/%s' % (repo.namespace_user.username, repo.name)
homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
repo_string)
@@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
event_data = {
'repository': repo_string,
- 'namespace': repo.namespace,
+ 'namespace': repo.namespace_user.username,
'name': repo.name,
'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string),
'homepage': homepage,
@@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
def build_notification_data(notification, event_data):
return {
'notification_uuid': notification.uuid,
- 'repository_namespace': notification.repository.namespace,
+ 'repository_namespace': notification.repository.namespace_user.username,
'repository_name': notification.repository.name,
'event_data': event_data
}
@@ -39,8 +39,9 @@ def build_notification_data(notification, event_data):
def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]):
event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage)
- notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name)
+ notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name,
+ event_name=event_name)
for notification in notifications:
notification_data = build_notification_data(notification, event_data)
- path = [repo.namespace, repo.name, event_name] + pathargs
+ path = [repo.namespace_user.username, repo.name, event_name] + pathargs
notification_queue.put(path, json.dumps(notification_data))
diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py
index 9650e79f6..589ebd06d 100644
--- a/endpoints/notificationmethod.py
+++ b/endpoints/notificationmethod.py
@@ -10,6 +10,7 @@ import re
from flask.ext.mail import Message
from app import mail, app, get_app_url
from data import model
+from workers.worker import JobException
logger = logging.getLogger(__name__)
@@ -19,6 +20,9 @@ class InvalidNotificationMethodException(Exception):
class CannotValidateNotificationMethodException(Exception):
pass
+class NotificationMethodPerformException(JobException):
+ pass
+
class NotificationMethod(object):
def __init__(self):
@@ -84,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod):
return (True, 'Unknown organization %s' % target_info['name'], None)
# Only repositories under the organization can cause notifications to that org.
- if target_info['name'] != repository.namespace:
+ if target_info['name'] != repository.namespace_user.username:
return (False, 'Organization name must match repository namespace')
return (True, None, [target])
@@ -92,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod):
# Lookup the team.
team = None
try:
- team = model.get_organization_team(repository.namespace, target_info['name'])
+ team = model.get_organization_team(repository.namespace_user.username, target_info['name'])
except model.InvalidTeamException:
# Probably deleted.
return (True, 'Unknown team %s' % target_info['name'], None)
@@ -105,19 +109,18 @@ class QuayNotificationMethod(NotificationMethod):
repository = notification.repository
if not repository:
# Probably deleted.
- return True
+ return
# Lookup the target user or team to which we'll send the notification.
config_data = json.loads(notification.config_json)
status, err_message, target_users = self.find_targets(repository, config_data)
if not status:
- return False
+ raise NotificationMethodPerformException(err_message)
# For each of the target users, create a notification.
for target_user in set(target_users or []):
model.create_notification(event_handler.event_name(), target_user,
metadata=notification_data['event_data'])
- return True
class EmailMethod(NotificationMethod):
@@ -130,7 +133,8 @@ class EmailMethod(NotificationMethod):
if not email:
raise CannotValidateNotificationMethodException('Missing e-mail address')
- record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email)
+ record = model.get_email_authorized_for_repo(repository.namespace_user.username,
+ repository.name, email)
if not record or not record.confirmed:
raise CannotValidateNotificationMethodException('The specified e-mail address '
'is not authorized to receive '
@@ -141,7 +145,7 @@ class EmailMethod(NotificationMethod):
config_data = json.loads(notification.config_json)
email = config_data.get('email', '')
if not email:
- return False
+ return
msg = Message(event_handler.get_summary(notification_data['event_data'], notification_data),
sender='support@quay.io',
@@ -153,9 +157,7 @@ class EmailMethod(NotificationMethod):
mail.send(msg)
except Exception as ex:
logger.exception('Email was unable to be sent: %s' % ex.message)
- return False
-
- return True
+ raise NotificationMethodPerformException(ex.message)
class WebhookMethod(NotificationMethod):
@@ -172,7 +174,7 @@ class WebhookMethod(NotificationMethod):
config_data = json.loads(notification.config_json)
url = config_data.get('url', '')
if not url:
- return False
+ return
payload = notification_data['event_data']
headers = {'Content-type': 'application/json'}
@@ -180,15 +182,14 @@ class WebhookMethod(NotificationMethod):
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
- logger.error('%s response for webhook to url: %s' % (resp.status_code,
- url))
- return False
+ error_message = '%s response for webhook to url: %s' % (resp.status_code, url)
+ logger.error(error_message)
+ logger.error(resp.content)
+ raise NotificationMethodPerformException(error_message)
except requests.exceptions.RequestException as ex:
logger.exception('Webhook was unable to be sent: %s' % ex.message)
- return False
-
- return True
+ raise NotificationMethodPerformException(ex.message)
class FlowdockMethod(NotificationMethod):
@@ -208,12 +209,12 @@ class FlowdockMethod(NotificationMethod):
config_data = json.loads(notification.config_json)
token = config_data.get('flow_api_token', '')
if not token:
- return False
+ return
- owner = model.get_user(notification.repository.namespace)
+ owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
- return False
+ return
url = 'https://api.flowdock.com/v1/messages/team_inbox/%s' % token
headers = {'Content-type': 'application/json'}
@@ -223,7 +224,8 @@ class FlowdockMethod(NotificationMethod):
'subject': event_handler.get_summary(notification_data['event_data'], notification_data),
'content': event_handler.get_message(notification_data['event_data'], notification_data),
'from_name': owner.username,
- 'project': notification.repository.namespace + ' ' + notification.repository.name,
+ 'project': (notification.repository.namespace_user.username + ' ' +
+ notification.repository.name),
'tags': ['#' + event_handler.event_name()],
'link': notification_data['event_data']['homepage']
}
@@ -231,16 +233,14 @@ class FlowdockMethod(NotificationMethod):
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
- logger.error('%s response for flowdock to url: %s' % (resp.status_code,
- url))
+ error_message = '%s response for flowdock to url: %s' % (resp.status_code, url)
+ logger.error(error_message)
logger.error(resp.content)
- return False
+ raise NotificationMethodPerformException(error_message)
except requests.exceptions.RequestException as ex:
logger.exception('Flowdock method was unable to be sent: %s' % ex.message)
- return False
-
- return True
+ raise NotificationMethodPerformException(ex.message)
class HipchatMethod(NotificationMethod):
@@ -265,12 +265,12 @@ class HipchatMethod(NotificationMethod):
room_id = config_data.get('room_id', '')
if not token or not room_id:
- return False
+ return
- owner = model.get_user(notification.repository.namespace)
+ owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
- return False
+ return
url = 'https://api.hipchat.com/v2/room/%s/notification?auth_token=%s' % (room_id, token)
@@ -293,16 +293,14 @@ class HipchatMethod(NotificationMethod):
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
- logger.error('%s response for hipchat to url: %s' % (resp.status_code,
- url))
+ error_message = '%s response for hipchat to url: %s' % (resp.status_code, url)
+ logger.error(error_message)
logger.error(resp.content)
- return False
+ raise NotificationMethodPerformException(error_message)
except requests.exceptions.RequestException as ex:
logger.exception('Hipchat method was unable to be sent: %s' % ex.message)
- return False
-
- return True
+ raise NotificationMethodPerformException(ex.message)
class SlackMethod(NotificationMethod):
@@ -334,12 +332,12 @@ class SlackMethod(NotificationMethod):
subdomain = config_data.get('subdomain', '')
if not token or not subdomain:
- return False
+ return
- owner = model.get_user(notification.repository.namespace)
+ owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
- return False
+ return
url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
@@ -370,13 +368,11 @@ class SlackMethod(NotificationMethod):
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
- logger.error('%s response for Slack to url: %s' % (resp.status_code,
- url))
+ error_message = '%s response for Slack to url: %s' % (resp.status_code, url)
+ logger.error(error_message)
logger.error(resp.content)
- return False
+ raise NotificationMethodPerformException(error_message)
except requests.exceptions.RequestException as ex:
logger.exception('Slack method was unable to be sent: %s' % ex.message)
- return False
-
- return True
+ raise NotificationMethodPerformException(ex.message)
diff --git a/endpoints/registry.py b/endpoints/registry.py
index 4713ddd75..5699f0db2 100644
--- a/endpoints/registry.py
+++ b/endpoints/registry.py
@@ -14,6 +14,7 @@ from util.http import abort, exact_abort
from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission)
from data import model
+from util import gzipstream
registry = Blueprint('registry', __name__)
@@ -193,21 +194,33 @@ def put_image_layer(namespace, repository, image_id):
# encoding (Gunicorn)
input_stream = request.environ['wsgi.input']
- # compute checksums
- csums = []
+ # Create a socket reader to read the input stream containing the layer data.
sr = SocketReader(input_stream)
+
+ # Add a handler that store the data in storage.
tmp, store_hndlr = store.temp_store_handler()
sr.add_handler(store_hndlr)
+
+ # Add a handler to compute the uncompressed size of the layer.
+ uncompressed_size_info, size_hndlr = gzipstream.calculate_size_handler()
+ sr.add_handler(size_hndlr)
+
+ # Add a handler which computes the checksum.
h, sum_hndlr = checksums.simple_checksum_handler(json_data)
sr.add_handler(sum_hndlr)
+
+ # Stream write the data to storage.
store.stream_write(repo_image.storage.locations, layer_path, sr)
+
+ # Append the computed checksum.
+ csums = []
csums.append('sha256:{0}'.format(h.hexdigest()))
try:
image_size = tmp.tell()
# Save the size of the image.
- model.set_image_size(image_id, namespace, repository, image_size)
+ model.set_image_size(image_id, namespace, repository, image_size, uncompressed_size_info.size)
tmp.seek(0)
csums.append(checksums.compute_tarsum(tmp, json_data))
@@ -451,12 +464,6 @@ def put_image_json(namespace, repository, image_id):
set_uploading_flag(repo_image, True)
- # We cleanup any old checksum in case it's a retry after a fail
- profile.debug('Cleanup old checksum')
- repo_image.storage.uncompressed_size = data.get('Size')
- repo_image.storage.checksum = None
- repo_image.storage.save()
-
# If we reach that point, it means that this is a new image or a retry
# on a failed push
# save the metadata
diff --git a/endpoints/trigger.py b/endpoints/trigger.py
index ae0b4b2b7..c7c47db79 100644
--- a/endpoints/trigger.py
+++ b/endpoints/trigger.py
@@ -36,6 +36,9 @@ class TriggerActivationException(Exception):
class TriggerDeactivationException(Exception):
pass
+class TriggerStartException(Exception):
+ pass
+
class ValidationRequestException(Exception):
pass
@@ -109,12 +112,19 @@ class BuildTrigger(object):
"""
raise NotImplementedError
- def manual_start(self, auth_token, config):
+ def manual_start(self, auth_token, config, run_parameters = None):
"""
Manually creates a repository build for this trigger.
"""
raise NotImplementedError
+ def list_field_values(self, auth_token, config, field_name):
+ """
+ Lists all values for the given custom trigger field. For example, a trigger might have a
+ field named "branches", and this method would return all branches.
+ """
+ raise NotImplementedError
+
@classmethod
def service_name(cls):
"""
@@ -345,14 +355,37 @@ class GithubBuildTrigger(BuildTrigger):
return GithubBuildTrigger._prepare_build(config, repo, commit_sha,
short_sha, ref)
- def manual_start(self, auth_token, config):
- source = config['build_source']
+ def manual_start(self, auth_token, config, run_parameters = None):
+ try:
+ source = config['build_source']
+ run_parameters = run_parameters or {}
- gh_client = self._get_client(auth_token)
- repo = gh_client.get_repo(source)
- master = repo.get_branch(repo.default_branch)
- master_sha = master.commit.sha
- short_sha = GithubBuildTrigger.get_display_name(master_sha)
- ref = 'refs/heads/%s' % repo.default_branch
+ gh_client = self._get_client(auth_token)
+ repo = gh_client.get_repo(source)
+ master = repo.get_branch(repo.default_branch)
+ master_sha = master.commit.sha
+ short_sha = GithubBuildTrigger.get_display_name(master_sha)
+ ref = 'refs/heads/%s' % (run_parameters.get('branch_name') or repo.default_branch)
- return self._prepare_build(config, repo, master_sha, short_sha, ref)
+ return self._prepare_build(config, repo, master_sha, short_sha, ref)
+ except GithubException as ghe:
+ raise TriggerStartException(ghe.data['message'])
+
+
+ def list_field_values(self, auth_token, config, field_name):
+ if field_name == 'branch_name':
+ gh_client = self._get_client(auth_token)
+ source = config['build_source']
+ repo = gh_client.get_repo(source)
+ branches = [branch.name for branch in repo.get_branches()]
+
+ if not repo.default_branch in branches:
+ branches.insert(0, repo.default_branch)
+
+ if branches[0] != repo.default_branch:
+ branches.remove(repo.default_branch)
+ branches.insert(0, repo.default_branch)
+
+ return branches
+
+ return None
diff --git a/endpoints/web.py b/endpoints/web.py
index 19f9bb7f1..63e463666 100644
--- a/endpoints/web.py
+++ b/endpoints/web.py
@@ -18,6 +18,7 @@ from endpoints.common import common_login, render_page_template, route_show_if,
from endpoints.csrf import csrf_protect, generate_csrf_token
from util.names import parse_repository_name
from util.gravatar import compute_hash
+from util.useremails import send_email_changed
from auth import scopes
import features
@@ -32,8 +33,8 @@ STATUS_TAGS = app.config['STATUS_TAGS']
@web.route('/', methods=['GET'], defaults={'path': ''})
@web.route('/organization/', methods=['GET'])
@no_cache
-def index(path):
- return render_page_template('index.html')
+def index(path, **kwargs):
+ return render_page_template('index.html', **kwargs)
@web.route('/500', methods=['GET'])
@@ -101,7 +102,7 @@ def superuser():
@web.route('/signin/')
@no_cache
-def signin():
+def signin(redirect=None):
return index('')
@@ -123,6 +124,13 @@ def new():
return index('')
+@web.route('/confirminvite')
+@no_cache
+def confirm_invite():
+ code = request.values['code']
+ return index('', code=code)
+
+
@web.route('/repository/', defaults={'path': ''})
@web.route('/repository/', methods=['GET'])
@no_cache
@@ -215,6 +223,7 @@ def receipt():
@web.route('/authrepoemail', methods=['GET'])
+@route_show_if(features.MAILING)
def confirm_repo_email():
code = request.values['code']
record = None
@@ -228,23 +237,27 @@ def confirm_repo_email():
Your E-mail address has been authorized to receive notifications for repository
%s/%s.
""" % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'],
- record.repository.namespace, record.repository.name,
- record.repository.namespace, record.repository.name)
+ record.repository.namespace_user.username, record.repository.name,
+ record.repository.namespace_user.username, record.repository.name)
return render_page_template('message.html', message=message)
@web.route('/confirm', methods=['GET'])
+@route_show_if(features.MAILING)
def confirm_email():
code = request.values['code']
user = None
new_email = None
try:
- user, new_email = model.confirm_user_email(code)
+ user, new_email, old_email = model.confirm_user_email(code)
except model.DataModelException as ex:
return render_page_template('confirmerror.html', error_message=ex.message)
+ if new_email:
+ send_email_changed(user.username, old_email, new_email)
+
common_login(user)
return redirect(url_for('web.user', tab='email')
diff --git a/initdb.py b/initdb.py
index 26ef28678..87208a8d6 100644
--- a/initdb.py
+++ b/initdb.py
@@ -51,7 +51,7 @@ def __gen_checksum(image_id):
def __gen_image_id(repo, image_num):
- str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num)
+ str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num)
h = hashlib.md5(str_to_hash)
return h.hexdigest() + h.hexdigest()
@@ -79,12 +79,12 @@ def __create_subtree(repo, structure, creator_username, parent):
creation_time = REFERENCE_DATE + timedelta(days=image_num)
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
command = json.dumps(command_list) if command_list else None
- new_image = model.set_image_metadata(docker_image_id, repo.namespace,
- repo.name, str(creation_time),
- 'no comment', command, parent)
+ new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name,
+ str(creation_time), 'no comment', command, parent)
- model.set_image_size(docker_image_id, repo.namespace, repo.name,
- random.randrange(1, 1024 * 1024 * 1024))
+ compressed_size = random.randrange(1, 1024 * 1024 * 1024)
+ model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, compressed_size,
+ int(compressed_size * 1.4))
# Populate the diff file
diff_path = store.image_file_diffs_path(new_image.storage.uuid)
@@ -100,7 +100,7 @@ def __create_subtree(repo, structure, creator_username, parent):
last_node_tags = [last_node_tags]
for tag_name in last_node_tags:
- model.create_or_update_tag(repo.namespace, repo.name, tag_name,
+ model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name,
new_image.docker_image_id)
for subtree in subtrees:
@@ -214,7 +214,11 @@ def initialize_database():
LogEntryKind.create(name='org_create_team')
LogEntryKind.create(name='org_delete_team')
+ LogEntryKind.create(name='org_invite_team_member')
+ LogEntryKind.create(name='org_delete_team_member_invite')
LogEntryKind.create(name='org_add_team_member')
+ LogEntryKind.create(name='org_team_member_invite_accepted')
+ LogEntryKind.create(name='org_team_member_invite_declined')
LogEntryKind.create(name='org_remove_team_member')
LogEntryKind.create(name='org_set_team_description')
LogEntryKind.create(name='org_set_team_role')
@@ -271,6 +275,7 @@ def initialize_database():
NotificationKind.create(name='over_private_usage')
NotificationKind.create(name='expiring_license')
NotificationKind.create(name='maintenance')
+ NotificationKind.create(name='org_team_invite')
NotificationKind.create(name='test_notification')
@@ -302,7 +307,7 @@ def populate_database():
new_user_2.verified = True
new_user_2.save()
- new_user_3 = model.create_user('freshuser', 'password', 'no@thanks.com')
+ new_user_3 = model.create_user('freshuser', 'password', 'jschorr+test@devtable.com')
new_user_3.verified = True
new_user_3.save()
@@ -323,7 +328,8 @@ def populate_database():
outside_org.verified = True
outside_org.save()
- model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}})
+ model.create_notification('test_notification', new_user_1,
+ metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}})
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
@@ -387,18 +393,20 @@ def populate_database():
})
trigger.save()
- repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
+ repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name)
job_config = {
'repository': repo,
'docker_tags': ['latest'],
'build_subdir': '',
}
- record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com')
+ record = model.create_email_authorization_for_repo(new_user_1.username, 'simple',
+ 'jschorr@devtable.com')
record.confirmed = True
record.save()
- model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com')
+ model.create_email_authorization_for_repo(new_user_1.username, 'simple',
+ 'jschorr+other@devtable.com')
build2 = model.create_repository_build(building, token, job_config,
'68daeebd-a5b9-457f-80a0-4363b882f8ea',
@@ -425,12 +433,12 @@ def populate_database():
model.create_robot('coolrobot', org)
- oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html',
- client_id='deadbeef')
+ oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
+ 'http://localhost:8000/o2c.html', client_id='deadbeef')
- oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html',
- client_id='deadpork',
- description = 'This is another test application')
+ oauth.create_application(org, 'Some Other Test App', 'http://quay.io',
+ 'http://localhost:8000/o2c.html', client_id='deadpork',
+ description='This is another test application')
model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin')
@@ -452,8 +460,8 @@ def populate_database():
reader_team = model.create_team('readers', org, 'member',
'Readers of orgrepo.')
- model.set_team_repo_permission(reader_team.name, org_repo.namespace,
- org_repo.name, 'read')
+ model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name,
+ 'read')
model.add_user_to_team(new_user_2, reader_team)
model.add_user_to_team(reader, reader_team)
@@ -475,12 +483,9 @@ def populate_database():
(2, [], 'latest17'),
(2, [], 'latest18'),])
- model.add_prototype_permission(org, 'read', activating_user=new_user_1,
- delegate_user=new_user_2)
- model.add_prototype_permission(org, 'read', activating_user=new_user_1,
- delegate_team=reader_team)
- model.add_prototype_permission(org, 'write', activating_user=new_user_2,
- delegate_user=new_user_1)
+ model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2)
+ model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team)
+ model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
diff --git a/static/css/quay.css b/static/css/quay.css
index 15c71634b..08d55c970 100644
--- a/static/css/quay.css
+++ b/static/css/quay.css
@@ -144,6 +144,15 @@ nav.navbar-default .navbar-nav>li>a.active {
max-width: 320px;
}
+.notification-view-element .right-controls button {
+ margin-left: 10px;
+}
+
+.notification-view-element .message i.fa {
+ margin-right: 6px;
+}
+
+
.notification-view-element .orginfo {
margin-top: 8px;
float: left;
@@ -3593,6 +3602,12 @@ p.editable:hover i {
white-space: nowrap;
}
+.tt-message {
+ padding: 10px;
+ font-size: 12px;
+ white-space: nowrap;
+}
+
.tt-suggestion p {
margin: 0;
}
@@ -4284,7 +4299,7 @@ pre.command:before {
}
.user-row.super-user td {
- background-color: #d9edf7;
+ background-color: #eeeeee;
}
.user-row .user-class {
@@ -4672,4 +4687,68 @@ i.slack-icon {
.external-notification-view-element:hover .side-controls button {
border: 1px solid #eee;
+}
+
+.member-listing {
+ width: 100%;
+}
+
+.member-listing .section-header {
+ color: #ccc;
+ margin-top: 20px;
+ margin-bottom: 10px;
+}
+
+.member-listing .gravatar {
+ vertical-align: middle;
+ margin-right: 10px;
+}
+
+.member-listing .entity-reference {
+ margin-bottom: 10px;
+ display: inline-block;
+}
+
+.member-listing .invite-listing {
+ margin-bottom: 10px;
+ display: inline-block;
+}
+
+.team-view .organization-header .popover {
+ max-width: none !important;
+}
+
+.team-view .organization-header .popover.bottom-right .arrow:after {
+ border-bottom-color: #f7f7f7;
+ top: 2px;
+}
+
+.team-view .organization-header .popover-content {
+ font-size: 14px;
+ padding-top: 6px;
+}
+
+.team-view .organization-header .popover-content input {
+ background: white;
+}
+
+.team-view .team-view-add-element .help-text {
+ font-size: 13px;
+ color: #ccc;
+ margin-top: 10px;
+}
+
+.team-view .organization-header .popover-content {
+ min-width: 500px;
+}
+
+#startTriggerDialog .trigger-description {
+ margin-bottom: 20px;
+ padding-bottom: 20px;
+ border-bottom: 1px solid #eee;
+}
+
+#startTriggerDialog #runForm .field-title {
+ width: 120px;
+ padding-right: 10px;
}
\ No newline at end of file
diff --git a/static/directives/entity-reference.html b/static/directives/entity-reference.html
index d01b100ee..ea65db875 100644
--- a/static/directives/entity-reference.html
+++ b/static/directives/entity-reference.html
@@ -7,15 +7,19 @@
- 
+ ![]()
{{entity.name}}
{{entity.name}}
-
-
+ ![]()
+
+
+
+
+
{{ getPrefix(entity.name) }}+{{ getShortenedName(entity.name) }}
diff --git a/static/directives/entity-search.html b/static/directives/entity-search.html
index fec00b393..63abb1528 100644
--- a/static/directives/entity-search.html
+++ b/static/directives/entity-search.html
@@ -5,7 +5,7 @@
ng-click="lazyLoad()">
-
+
-
diff --git a/static/directives/external-login-button.html b/static/directives/external-login-button.html
index cc1d39bbd..d241089d6 100644
--- a/static/directives/external-login-button.html
+++ b/static/directives/external-login-button.html
@@ -1,6 +1,6 @@
-
+
Sign In with GitHub
Attach to GitHub Account
@@ -8,7 +8,7 @@
-
+
Sign In with Google
Attach to Google Account
diff --git a/static/directives/logs-view.html b/static/directives/logs-view.html
index 5ccea2f3f..28509b897 100644
--- a/static/directives/logs-view.html
+++ b/static/directives/logs-view.html
@@ -3,7 +3,7 @@
Usage Logs
-
+
From
diff --git a/static/directives/manual-trigger-build-dialog.html b/static/directives/manual-trigger-build-dialog.html
new file mode 100644
index 000000000..e82879e8a
--- /dev/null
+++ b/static/directives/manual-trigger-build-dialog.html
@@ -0,0 +1,38 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/static/directives/notification-view.html b/static/directives/notification-view.html
index f03133e55..fbdb4b419 100644
--- a/static/directives/notification-view.html
+++ b/static/directives/notification-view.html
@@ -7,10 +7,13 @@
{{ notification.organization }}
- {{ parseDate(notification.created) | date:'medium'}}
+ {{ parseDate(notification.created) | date:'medium'}}
diff --git a/static/directives/prototype-manager.html b/static/directives/prototype-manager.html
index 7143b13b3..35e5b9c0c 100644
--- a/static/directives/prototype-manager.html
+++ b/static/directives/prototype-manager.html
@@ -3,7 +3,7 @@
- Default permissions provide a means of specifying additional permissions that should be granted automatically to a repository.
+ Default permissions provide a means of specifying additional permissions that should be granted automatically to a repository when it is created.
diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html
index de67c18f6..59551b51c 100644
--- a/static/directives/signin-form.html
+++ b/static/directives/signin-form.html
@@ -1,5 +1,6 @@
-
-
+
-
+
-
+
+
-
+
Build Triggers
@@ -378,6 +378,12 @@
counter="showNewNotificationCounter"
notification-created="handleNotificationCreated(notification)">
+
+
+
diff --git a/static/partials/signin.html b/static/partials/signin.html
index 4aac6cb7e..2a1a9563d 100644
--- a/static/partials/signin.html
+++ b/static/partials/signin.html
@@ -1,7 +1,7 @@
-
+
diff --git a/static/partials/super-user.html b/static/partials/super-user.html
index bc21f5c94..a12f9f5b2 100644
--- a/static/partials/super-user.html
+++ b/static/partials/super-user.html
@@ -1,4 +1,4 @@
-
+
This panel provides administrator access to super users of this installation of the registry. Super users can be managed in the configuration for this installation.
@@ -10,18 +10,64 @@
-
Users
+ -
+ Create User
+
+ -
+ System Logs
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Username
+ E-mail address
+ Temporary Password
+
+
+
+ {{ created_user.username }}
+ {{ created_user.email }}
+ {{ created_user.password }}
+
+
+
+
+
{{ usersError }}
-
+
@@ -37,8 +83,7 @@
Username
E-mail address
-
{{ current_user.email }}
-
- Super user
-
-
-
+
+
+
-
+
-
Change Password
+
+ Send Recovery Email
+
Delete User
diff --git a/static/partials/team-view.html b/static/partials/team-view.html
index b55721455..90ef77696 100644
--- a/static/partials/team-view.html
+++ b/static/partials/team-view.html
@@ -1,40 +1,92 @@
-
+
+
+
+
+
-
- Team Members
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+ Team Members
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Robot Accounts
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Invited To Join
+
+
+
+
+
+
+
+
+ ![]()
+ {{ member.email }}
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html
index 7d6dd8dfb..a51119531 100644
--- a/static/partials/user-admin.html
+++ b/static/partials/user-admin.html
@@ -122,7 +122,7 @@
-
+
Change e-mail address
diff --git a/templates/ologinerror.html b/templates/ologinerror.html
index 304b7f554..74f51987e 100644
--- a/templates/ologinerror.html
+++ b/templates/ologinerror.html
@@ -8,17 +8,19 @@
- There was an error logging in with {{ service_name }}.
+ There was an error logging in with {{ service_name }}.
{% if error_message %}
{{ error_message }}
{% endif %}
+ {% if user_creation %}
Please register using the registration form to continue.
You will be able to connect your account to your Quay.io account
in the user settings.
+ {% endif %}
diff --git a/test/data/test.db b/test/data/test.db
index 2c16ee353..4be5d0b90 100644
Binary files a/test/data/test.db and b/test/data/test.db differ
diff --git a/test/test_api_security.py b/test/test_api_security.py
index 1f4a18a84..b0d9c271c 100644
--- a/test/test_api_security.py
+++ b/test/test_api_security.py
@@ -1,14 +1,16 @@
import unittest
import json
+import datetime
from urllib import urlencode
from urlparse import urlparse, urlunparse, parse_qs
from app import app
+from data import model
from initdb import setup_database_for_testing, finished_database_for_testing
from endpoints.api import api_bp, api
-from endpoints.api.team import TeamMember, TeamMemberList, OrganizationTeam
+from endpoints.api.team import TeamMember, TeamMemberList, OrganizationTeam, TeamMemberInvite
from endpoints.api.tag import RepositoryTagImages, RepositoryTag
from endpoints.api.search import FindRepositories, EntitySearch
from endpoints.api.image import RepositoryImageChanges, RepositoryImage, RepositoryImageList
@@ -19,7 +21,7 @@ from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobo
from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs,
TriggerBuildList, ActivateBuildTrigger, BuildTrigger,
- BuildTriggerList, BuildTriggerAnalyze)
+ BuildTriggerList, BuildTriggerAnalyze, BuildTriggerFieldValues)
from endpoints.api.repoemail import RepositoryAuthorizedEmail
from endpoints.api.repositorynotification import RepositoryNotification, RepositoryNotificationList
from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Recovery, Signout,
@@ -40,7 +42,8 @@ from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repos
from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission,
RepositoryTeamPermissionList, RepositoryUserPermissionList)
-from endpoints.api.superuser import SuperUserLogs, SuperUserList, SuperUserManagement
+from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserManagement,
+ SuperUserSendRecoveryEmail)
try:
@@ -75,7 +78,9 @@ class ApiTestCase(unittest.TestCase):
with client.session_transaction() as sess:
if auth_username:
- sess['user_id'] = auth_username
+ loaded = model.get_user(auth_username)
+ sess['user_id'] = loaded.id
+ sess['login_time'] = datetime.datetime.now()
sess[CSRF_TOKEN_KEY] = CSRF_TOKEN
# Restore the teardown functions
@@ -510,13 +515,13 @@ class TestUser(ApiTestCase):
self._run_test('PUT', 401, None, {})
def test_put_freshuser(self):
- self._run_test('PUT', 401, 'freshuser', {})
+ self._run_test('PUT', 200, 'freshuser', {})
def test_put_reader(self):
- self._run_test('PUT', 401, 'reader', {})
+ self._run_test('PUT', 200, 'reader', {})
def test_put_devtable(self):
- self._run_test('PUT', 401, 'devtable', {})
+ self._run_test('PUT', 200, 'devtable', {})
def test_post_anonymous(self):
self._run_test('POST', 400, None, {u'username': 'T946', u'password': '0SG4', u'email': 'MENT'})
@@ -1061,6 +1066,62 @@ class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase):
def test_post_devtable(self):
self._run_test('POST', 404, 'devtable', {'config': {}})
+class TestBuildTriggerFieldValuesSwo1PublicPublicrepo(ApiTestCase):
+ def setUp(self):
+ ApiTestCase.setUp(self)
+ self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="public/publicrepo",
+ field_name="test_field")
+
+ def test_get_anonymous(self):
+ self._run_test('GET', 401, None, {})
+
+ def test_get_freshuser(self):
+ self._run_test('GET', 403, 'freshuser', {})
+
+ def test_get_reader(self):
+ self._run_test('GET', 403, 'reader', {})
+
+ def test_get_devtable(self):
+ self._run_test('GET', 403, 'devtable', {})
+
+
+class TestBuildTriggerFieldValuesSwo1DevtableShared(ApiTestCase):
+ def setUp(self):
+ ApiTestCase.setUp(self)
+ self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="devtable/shared",
+ field_name="test_field")
+
+ def test_get_anonymous(self):
+ self._run_test('GET', 401, None, {})
+
+ def test_get_freshuser(self):
+ self._run_test('GET', 403, 'freshuser', {})
+
+ def test_get_reader(self):
+ self._run_test('GET', 403, 'reader', {})
+
+ def test_get_devtable(self):
+ self._run_test('GET', 404, 'devtable', {'config': {}})
+
+
+class TestBuildTriggerFieldValuesSwo1BuynlargeOrgrepo(ApiTestCase):
+ def setUp(self):
+ ApiTestCase.setUp(self)
+ self._set_url(BuildTriggerFieldValues, trigger_uuid="SWO1", repository="buynlarge/orgrepo",
+ field_name="test_field")
+
+ def test_get_anonymous(self):
+ self._run_test('GET', 401, None, {})
+
+ def test_get_freshuser(self):
+ self._run_test('GET', 403, 'freshuser', {})
+
+ def test_get_reader(self):
+ self._run_test('GET', 403, 'reader', {})
+
+ def test_get_devtable(self):
+ self._run_test('GET', 404, 'devtable', {'config': {}})
+
class TestBuildTriggerSources831cPublicPublicrepo(ApiTestCase):
def setUp(self):
@@ -1292,7 +1353,7 @@ class TestActivateBuildTrigger0byeDevtableShared(ApiTestCase):
self._run_test('POST', 403, 'reader', None)
def test_post_devtable(self):
- self._run_test('POST', 404, 'devtable', None)
+ self._run_test('POST', 404, 'devtable', {})
class TestActivateBuildTrigger0byeBuynlargeOrgrepo(ApiTestCase):
@@ -1310,7 +1371,7 @@ class TestActivateBuildTrigger0byeBuynlargeOrgrepo(ApiTestCase):
self._run_test('POST', 403, 'reader', None)
def test_post_devtable(self):
- self._run_test('POST', 404, 'devtable', None)
+ self._run_test('POST', 404, 'devtable', {})
class TestBuildTriggerAnalyze0byePublicPublicrepo(ApiTestCase):
@@ -3527,13 +3588,61 @@ class TestSuperUserLogs(ApiTestCase):
self._run_test('GET', 200, 'devtable', None)
+class TestSuperUserSendRecoveryEmail(ApiTestCase):
+ def setUp(self):
+ ApiTestCase.setUp(self)
+ self._set_url(SuperUserSendRecoveryEmail, username='someuser')
+
+ def test_post_anonymous(self):
+ self._run_test('POST', 401, None, None)
+
+ def test_post_freshuser(self):
+ self._run_test('POST', 403, 'freshuser', None)
+
+ def test_post_reader(self):
+ self._run_test('POST', 403, 'reader', None)
+
+ def test_post_devtable(self):
+ self._run_test('POST', 404, 'devtable', None)
+
+
+class TestTeamMemberInvite(ApiTestCase):
+ def setUp(self):
+ ApiTestCase.setUp(self)
+ self._set_url(TeamMemberInvite, code='foobarbaz')
+
+ def test_put_anonymous(self):
+ self._run_test('PUT', 401, None, None)
+
+ def test_put_freshuser(self):
+ self._run_test('PUT', 400, 'freshuser', None)
+
+ def test_put_reader(self):
+ self._run_test('PUT', 400, 'reader', None)
+
+ def test_put_devtable(self):
+ self._run_test('PUT', 400, 'devtable', None)
+
+ def test_delete_anonymous(self):
+ self._run_test('DELETE', 401, None, None)
+
+ def test_delete_freshuser(self):
+ self._run_test('DELETE', 400, 'freshuser', None)
+
+ def test_delete_reader(self):
+ self._run_test('DELETE', 400, 'reader', None)
+
+ def test_delete_devtable(self):
+ self._run_test('DELETE', 400, 'devtable', None)
+
+
class TestSuperUserList(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(SuperUserList)
def test_get_anonymous(self):
- self._run_test('GET', 403, None, None)
+ self._run_test('GET', 401, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
@@ -3545,14 +3654,13 @@ class TestSuperUserList(ApiTestCase):
self._run_test('GET', 200, 'devtable', None)
-
class TestSuperUserManagement(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(SuperUserManagement, username='freshuser')
def test_get_anonymous(self):
- self._run_test('GET', 403, None, None)
+ self._run_test('GET', 401, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
@@ -3565,7 +3673,7 @@ class TestSuperUserManagement(ApiTestCase):
def test_put_anonymous(self):
- self._run_test('PUT', 403, None, {})
+ self._run_test('PUT', 401, None, {})
def test_put_freshuser(self):
self._run_test('PUT', 403, 'freshuser', {})
@@ -3578,7 +3686,7 @@ class TestSuperUserManagement(ApiTestCase):
def test_delete_anonymous(self):
- self._run_test('DELETE', 403, None, None)
+ self._run_test('DELETE', 401, None, None)
def test_delete_freshuser(self):
self._run_test('DELETE', 403, 'freshuser', None)
diff --git a/test/test_api_usage.py b/test/test_api_usage.py
index 004f20651..05fe525be 100644
--- a/test/test_api_usage.py
+++ b/test/test_api_usage.py
@@ -1,3 +1,5 @@
+# coding=utf-8
+
import unittest
import json as py_json
@@ -11,7 +13,7 @@ from app import app
from initdb import setup_database_for_testing, finished_database_for_testing
from data import model, database
-from endpoints.api.team import TeamMember, TeamMemberList, OrganizationTeam
+from endpoints.api.team import TeamMember, TeamMemberList, TeamMemberInvite, OrganizationTeam
from endpoints.api.tag import RepositoryTagImages, RepositoryTag
from endpoints.api.search import FindRepositories, EntitySearch
from endpoints.api.image import RepositoryImage, RepositoryImageList
@@ -20,7 +22,7 @@ from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobo
RegenerateUserRobot, RegenerateOrgRobot)
from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs,
TriggerBuildList, ActivateBuildTrigger, BuildTrigger,
- BuildTriggerList, BuildTriggerAnalyze)
+ BuildTriggerList, BuildTriggerAnalyze, BuildTriggerFieldValues)
from endpoints.api.repoemail import RepositoryAuthorizedEmail
from endpoints.api.repositorynotification import RepositoryNotification, RepositoryNotificationList
from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Signout, Signin, User,
@@ -131,6 +133,10 @@ class ApiTestCase(unittest.TestCase):
def deleteResponse(self, resource_name, params={}, expected_code=204):
rv = self.app.delete(self.url_for(resource_name, params))
+
+ if rv.status_code != expected_code:
+ print 'Mismatch data for resource DELETE %s: %s' % (resource_name, rv.data)
+
self.assertEquals(rv.status_code, expected_code)
return rv.data
@@ -162,6 +168,13 @@ class ApiTestCase(unittest.TestCase):
parsed = py_json.loads(data)
return parsed
+ def assertInTeam(self, data, membername):
+ for memberData in data['members']:
+ if memberData['name'] == membername:
+ return
+
+ self.fail(membername + ' not found in team: ' + py_json.dumps(data))
+
def login(self, username, password='password'):
return self.postJsonResponse(Signin, data=dict(username=username, password=password))
@@ -328,6 +341,12 @@ class TestChangeUserDetails(ApiTestCase):
data=dict(password='newpasswordiscool'))
self.login(READ_ACCESS_USER, password='newpasswordiscool')
+ def test_changepassword_unicode(self):
+ self.login(READ_ACCESS_USER)
+ self.putJsonResponse(User,
+ data=dict(password=u'someunicode北京市pass'))
+ self.login(READ_ACCESS_USER, password=u'someunicode北京市pass')
+
def test_changeeemail(self):
self.login(READ_ACCESS_USER)
@@ -375,10 +394,30 @@ class TestCreateNewUser(ApiTestCase):
self.assertEquals('Invalid username auserName: Username must match expression [a-z0-9_]+', json['error_description'])
def test_createuser(self):
- data = self.postResponse(User,
+ data = self.postJsonResponse(User,
data=NEW_USER_DETAILS,
- expected_code=201)
- self.assertEquals('"Created"', data)
+ expected_code=200)
+ self.assertEquals(True, data['awaiting_verification'])
+
+ def test_createuser_withteaminvite(self):
+ inviter = model.get_user(ADMIN_ACCESS_USER)
+ team = model.get_organization_team(ORGANIZATION, 'owners')
+ invite = model.add_or_invite_to_team(inviter, team, None, 'foo@example.com')
+
+ details = {
+ 'invite_code': invite.invite_token
+ }
+ details.update(NEW_USER_DETAILS);
+
+ data = self.postJsonResponse(User, data=details, expected_code=200)
+ self.assertEquals(True, data['awaiting_verification'])
+
+ # Make sure the user was added to the team.
+ self.login(ADMIN_ACCESS_USER)
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='owners'))
+ self.assertInTeam(json, NEW_USER_DETAILS['username'])
class TestSignout(ApiTestCase):
@@ -741,16 +780,43 @@ class TestGetOrganizationTeamMembers(ApiTestCase):
params=dict(orgname=ORGANIZATION,
teamname='readers'))
- assert READ_ACCESS_USER in json['members']
+ self.assertEquals(READ_ACCESS_USER, json['members'][1]['name'])
class TestUpdateOrganizationTeamMember(ApiTestCase):
- def test_addmember(self):
+ def test_addmember_alreadyteammember(self):
self.login(ADMIN_ACCESS_USER)
+ membername = READ_ACCESS_USER
+ self.putResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='readers',
+ membername=membername),
+ expected_code=400)
+
+
+ def test_addmember_orgmember(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ membername = READ_ACCESS_USER
+ self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='owners',
+ membername=membername))
+
+ # Verify the user was added to the team.
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='owners'))
+
+ self.assertInTeam(json, membername)
+
+
+ def test_addmember_robot(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ membername = ORGANIZATION + '+coolrobot'
self.putJsonResponse(TeamMember,
params=dict(orgname=ORGANIZATION, teamname='readers',
- membername=NO_ACCESS_USER))
+ membername=membername))
# Verify the user was added to the team.
@@ -758,10 +824,168 @@ class TestUpdateOrganizationTeamMember(ApiTestCase):
params=dict(orgname=ORGANIZATION,
teamname='readers'))
- assert NO_ACCESS_USER in json['members']
+ self.assertInTeam(json, membername)
+
+
+ def test_addmember_invalidrobot(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ membername = 'freshuser+anotherrobot'
+ self.putResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='readers',
+ membername=membername),
+ expected_code=400)
+
+
+ def test_addmember_nonorgmember(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ membername = NO_ACCESS_USER
+ response = self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='owners',
+ membername=membername))
+
+
+ self.assertEquals(True, response['invited'])
+
+ # Make sure the user is not (yet) part of the team.
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='readers'))
+
+ for member in json['members']:
+ self.assertNotEqual(membername, member['name'])
+
+
+class TestAcceptTeamMemberInvite(ApiTestCase):
+ def assertInTeam(self, data, membername):
+ for memberData in data['members']:
+ if memberData['name'] == membername:
+ return
+
+ self.fail(membername + ' not found in team: ' + json.dumps(data))
+
+ def test_accept(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ # Create the invite.
+ membername = NO_ACCESS_USER
+ response = self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='owners',
+ membername=membername))
+
+ self.assertEquals(True, response['invited'])
+
+ # Login as the user.
+ self.login(membername)
+
+ # Accept the invite.
+ user = model.get_user(membername)
+ invites = list(model.lookup_team_invites(user))
+ self.assertEquals(1, len(invites))
+
+ self.putJsonResponse(TeamMemberInvite,
+ params=dict(code=invites[0].invite_token))
+
+ # Verify the user is now on the team.
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='owners'))
+
+ self.assertInTeam(json, membername)
+
+ # Verify the accept now fails.
+ self.putResponse(TeamMemberInvite,
+ params=dict(code=invites[0].invite_token),
+ expected_code=400)
+
+
+
+class TestDeclineTeamMemberInvite(ApiTestCase):
+ def test_decline_wronguser(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ # Create the invite.
+ membername = NO_ACCESS_USER
+ response = self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='owners',
+ membername=membername))
+
+ self.assertEquals(True, response['invited'])
+
+ # Try to decline the invite.
+ user = model.get_user(membername)
+ invites = list(model.lookup_team_invites(user))
+ self.assertEquals(1, len(invites))
+
+ self.deleteResponse(TeamMemberInvite,
+ params=dict(code=invites[0].invite_token),
+ expected_code=400)
+
+
+ def test_decline(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ # Create the invite.
+ membername = NO_ACCESS_USER
+ response = self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='owners',
+ membername=membername))
+
+ self.assertEquals(True, response['invited'])
+
+ # Login as the user.
+ self.login(membername)
+
+ # Decline the invite.
+ user = model.get_user(membername)
+ invites = list(model.lookup_team_invites(user))
+ self.assertEquals(1, len(invites))
+
+ self.deleteResponse(TeamMemberInvite,
+ params=dict(code=invites[0].invite_token))
+
+ # Make sure the invite was deleted.
+ self.deleteResponse(TeamMemberInvite,
+ params=dict(code=invites[0].invite_token),
+ expected_code=400)
class TestDeleteOrganizationTeamMember(ApiTestCase):
+ def test_deletememberinvite(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ membername = NO_ACCESS_USER
+ response = self.putJsonResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='readers',
+ membername=membername))
+
+
+ self.assertEquals(True, response['invited'])
+
+ # Verify the invite was added.
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='readers',
+ includePending=True))
+
+ assert len(json['members']) == 3
+
+ # Delete the invite.
+ self.deleteResponse(TeamMember,
+ params=dict(orgname=ORGANIZATION, teamname='readers',
+ membername=membername))
+
+
+ # Verify the user was removed from the team.
+ json = self.getJsonResponse(TeamMemberList,
+ params=dict(orgname=ORGANIZATION,
+ teamname='readers',
+ includePending=True))
+
+ assert len(json['members']) == 2
+
+
def test_deletemember(self):
self.login(ADMIN_ACCESS_USER)
@@ -775,7 +999,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase):
params=dict(orgname=ORGANIZATION,
teamname='readers'))
- assert not READ_ACCESS_USER in json['members']
+ assert len(json['members']) == 1
class TestCreateRepo(ApiTestCase):
@@ -1205,6 +1429,7 @@ class TestListAndGetImage(ApiTestCase):
params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
assert len(json['images']) > 0
+
for image in json['images']:
assert 'id' in image
assert 'tags' in image
@@ -1212,7 +1437,6 @@ class TestListAndGetImage(ApiTestCase):
assert 'comment' in image
assert 'command' in image
assert 'ancestors' in image
- assert 'dbid' in image
assert 'size' in image
ijson = self.getJsonResponse(RepositoryImage,
@@ -1810,7 +2034,7 @@ class FakeBuildTrigger(BuildTriggerBase):
config['active'] = False
return config
- def manual_start(self, auth_token, config):
+ def manual_start(self, auth_token, config, run_parameters=None):
return ('foo', ['bar'], 'build-name', 'subdir')
def dockerfile_url(self, auth_token, config):
@@ -1822,6 +2046,12 @@ class FakeBuildTrigger(BuildTriggerBase):
return config['dockerfile']
+ def list_field_values(self, auth_token, config, field_name):
+ if field_name == 'test_field':
+ return [1, 2, 3]
+
+ return None
+
class TestBuildTriggers(ApiTestCase):
def test_list_build_triggers(self):
@@ -1994,9 +2224,22 @@ class TestBuildTriggers(ApiTestCase):
data={'config': trigger_config},
expected_code=400)
+ # Retrieve values for a field.
+ result = self.getJsonResponse(BuildTriggerFieldValues,
+ params=dict(repository=ADMIN_ACCESS_USER + '/simple',
+ trigger_uuid=trigger.uuid, field_name="test_field"))
+
+ self.assertEquals(result['values'], [1, 2, 3])
+
+ self.getResponse(BuildTriggerFieldValues,
+ params=dict(repository=ADMIN_ACCESS_USER + '/simple',
+ trigger_uuid=trigger.uuid, field_name="another_field"),
+ expected_code = 404)
+
# Start a manual build.
start_json = self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
+ data=dict(),
expected_code=201)
assert 'id' in start_json
@@ -2061,6 +2304,7 @@ class TestBuildTriggers(ApiTestCase):
# Start a manual build.
start_json = self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
+ data=dict(),
expected_code=201)
assert 'id' in start_json
@@ -2120,7 +2364,7 @@ class TestSuperUserManagement(ApiTestCase):
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
- self.assertEquals('no@thanks.com', json['email'])
+ self.assertEquals('jschorr+test@devtable.com', json['email'])
self.assertEquals(False, json['super_user'])
def test_delete_user(self):
@@ -2143,7 +2387,7 @@ class TestSuperUserManagement(ApiTestCase):
# Verify the user exists.
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
- self.assertEquals('no@thanks.com', json['email'])
+ self.assertEquals('jschorr+test@devtable.com', json['email'])
# Update the user.
self.putJsonResponse(SuperUserManagement, params=dict(username='freshuser'), data=dict(email='foo@bar.com'))
diff --git a/test/testconfig.py b/test/testconfig.py
index 35c96a803..46288de7e 100644
--- a/test/testconfig.py
+++ b/test/testconfig.py
@@ -34,6 +34,7 @@ class TestConfig(DefaultConfig):
FEATURE_SUPER_USERS = True
FEATURE_BILLING = True
+ FEATURE_MAILING = True
SUPER_USERS = ['devtable']
LICENSE_USER_LIMIT = 500
diff --git a/test/testlogs.py b/test/testlogs.py
index 7fd9eac21..cd4ea6c9d 100644
--- a/test/testlogs.py
+++ b/test/testlogs.py
@@ -45,8 +45,8 @@ class TestBuildLogs(RedisBuildLogs):
'pull_completion': 0.0,
}
- def __init__(self, redis_host, namespace, repository, test_build_id, allow_delegate=True):
- super(TestBuildLogs, self).__init__(redis_host)
+ def __init__(self, redis_config, namespace, repository, test_build_id, allow_delegate=True):
+ super(TestBuildLogs, self).__init__(redis_config)
self.namespace = namespace
self.repository = repository
self.test_build_id = test_build_id
diff --git a/tools/auditancestry.py b/tools/auditancestry.py
index 95e082642..59d636836 100644
--- a/tools/auditancestry.py
+++ b/tools/auditancestry.py
@@ -1,7 +1,7 @@
import logging
import json
-from data.database import Image, ImageStorage, Repository, configure
+from data.database import Image, ImageStorage, Repository, User, configure
from data import model
from app import app, storage as store
@@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
query = (Image
- .select(Image, ImageStorage, Repository)
- .join(ImageStorage)
- .switch(Image)
- .join(Repository)
- .where(ImageStorage.uploading == False))
+ .select(Image, ImageStorage, Repository, User)
+ .join(ImageStorage)
+ .switch(Image)
+ .join(Repository)
+ .join(User)
+ .where(ImageStorage.uploading == False))
bad_count = 0
good_count = 0
def resolve_or_create(repo, docker_image_id, new_ancestry):
- existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
+ existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id)
if existing:
logger.debug('Found existing image: %s, %s', existing.id, docker_image_id)
return existing
@@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
return created
except ImageStorage.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s'
- logger.error(msg, docker_image_id, repo.namespace)
+ logger.error(msg, docker_image_id, repo.namespace_user.username)
raise RuntimeError()
@@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors):
cant_fix = []
for img in query:
try:
- with_locations = model.get_repo_image(img.repository.namespace, img.repository.name,
- img.docker_image_id)
+ with_locations = model.get_repo_image(img.repository.namespace_user.username,
+ img.repository.name, img.docker_image_id)
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(with_locations.storage.locations, ancestry_storage):
- full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:]
+ full_ancestry = json.loads(store.get_content(with_locations.storage.locations,
+ ancestry_storage))[1:]
full_ancestry.reverse()
- ancestor_dbids = [int(anc_id)
- for anc_id in img.ancestors.split('/')[1:-1]]
+ ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]]
if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids):
- logger.error('Image has incomplete ancestry: %s, %s, %s, %s' %
- (img.id, img.docker_image_id, full_ancestry,
- ancestor_dbids))
+ logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id,
+ full_ancestry, ancestor_dbids)
fixed_ancestry = '/'
for ancestor in full_ancestry:
@@ -99,5 +99,5 @@ for img in query:
len(cant_fix))
for cant in cant_fix:
- logger.error('Unable to fix %s in repo %s/%s', cant.id,
- cant.repository.namespace, cant.repository.name)
\ No newline at end of file
+ logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username,
+ cant.repository.name)
diff --git a/tools/orphans.py b/tools/orphans.py
index 17babf31a..a8990f79b 100644
--- a/tools/orphans.py
+++ b/tools/orphans.py
@@ -1,22 +1,15 @@
-from data.database import Image
-from app import app, storage as store
+from data.database import Image, ImageStorage
+from peewee import JOIN_LEFT_OUTER, fn
+from app import app
-live_image_id_set = set()
+orphaned = (ImageStorage
+ .select()
+ .where(ImageStorage.uploading == False)
+ .join(Image, JOIN_LEFT_OUTER)
+ .group_by(ImageStorage)
+ .having(fn.Count(Image.id) == 0))
-for image in Image.select():
- live_image_id_set.add(image.docker_image_id)
-
-storage_image_id_set = set()
-for customer in store.list_directory('images/'):
- for repo in store.list_directory(customer):
- for image in store.list_directory(repo):
- storage_image_id_set.add(image.split('/')[-1])
-
-orphans = storage_image_id_set.difference(live_image_id_set)
-missing_image_data = live_image_id_set.difference(storage_image_id_set)
-
-for orphan in orphans:
- print "Orphan: %s" % orphan
-
-for missing in missing_image_data:
- print "Missing: %s" % missing
+counter = 0
+for orphan in orphaned:
+ counter += 1
+ print orphan.uuid
diff --git a/tools/relationships.py b/tools/relationships.py
index d701a18ce..c2cad5de5 100644
--- a/tools/relationships.py
+++ b/tools/relationships.py
@@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile:
outfile.write('digraph relationships {\n')
for repo in Repository.select():
- ns = fix_ident(repo.namespace)
+ ns = fix_ident(repo.namespace_user.username)
outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns))
teams_in_orgs = set()
diff --git a/tools/uncompressedsize.py b/tools/uncompressedsize.py
new file mode 100644
index 000000000..53bb74126
--- /dev/null
+++ b/tools/uncompressedsize.py
@@ -0,0 +1,75 @@
+import logging
+import zlib
+
+from data import model
+from data.database import ImageStorage
+from app import app, storage as store
+from data.database import db, db_random_func
+from util.gzipstream import ZLIB_GZIP_WINDOW
+
+
+logger = logging.getLogger(__name__)
+
+
+CHUNK_SIZE = 5 * 1024 * 1024
+
+
+def backfill_sizes_from_data():
+ while True:
+ # Load the record from the DB.
+ batch_ids = list(ImageStorage
+ .select(ImageStorage.uuid)
+ .where(ImageStorage.uncompressed_size >> None,
+ ImageStorage.uploading == False)
+ .limit(100)
+ .order_by(db_random_func()))
+ if len(batch_ids) == 0:
+ # We're done!
+ return
+
+ for record in batch_ids:
+ uuid = record.uuid
+
+ try:
+ with_locs = model.get_storage_by_uuid(uuid)
+ if with_locs.uncompressed_size is not None:
+ logger.debug('Somebody else already filled this in for us: %s', uuid)
+ continue
+
+ # Read the layer from backing storage and calculate the uncompressed size.
+ logger.debug('Loading data: %s (%s bytes)', uuid, with_locs.image_size)
+ decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW)
+
+ uncompressed_size = 0
+ with store.stream_read_file(with_locs.locations, store.image_layer_path(uuid)) as stream:
+ while True:
+ current_data = stream.read(CHUNK_SIZE)
+ if len(current_data) == 0:
+ break
+
+ uncompressed_size += len(decompressor.decompress(current_data))
+
+ # Write the size to the image storage. We do so under a transaction AFTER checking to
+ # make sure the image storage still exists and has not changed.
+ logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size)
+ with app.config['DB_TRANSACTION_FACTORY'](db):
+ current_record = model.get_storage_by_uuid(uuid)
+
+ if not current_record.uploading and current_record.uncompressed_size == None:
+ current_record.uncompressed_size = uncompressed_size
+ current_record.save()
+ else:
+ logger.debug('Somebody else already filled this in for us, after we did the work: %s',
+ uuid)
+
+ except model.InvalidImageException:
+ logger.warning('Storage with uuid no longer exists: %s', uuid)
+ except MemoryError:
+ logger.warning('MemoryError on %s', uuid)
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.DEBUG)
+ logging.getLogger('boto').setLevel(logging.CRITICAL)
+ logging.getLogger('peewee').setLevel(logging.CRITICAL)
+
+ backfill_sizes_from_data()
diff --git a/util/gzipstream.py b/util/gzipstream.py
new file mode 100644
index 000000000..8f5aa9f6d
--- /dev/null
+++ b/util/gzipstream.py
@@ -0,0 +1,28 @@
+"""
+Defines utility methods for working with gzip streams.
+"""
+
+import zlib
+
+# Window size for decompressing GZIP streams.
+# This results in ZLIB automatically detecting the GZIP headers.
+# http://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760
+ZLIB_GZIP_WINDOW = zlib.MAX_WBITS | 32
+
+class SizeInfo(object):
+ def __init__(self):
+ self.size = 0
+
+def calculate_size_handler():
+ """ Returns an object and a SocketReader handler. The handler will gunzip the data it receives,
+ adding the size found to the object.
+ """
+
+ size_info = SizeInfo()
+
+ decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW)
+
+ def fn(buf):
+ size_info.size += len(decompressor.decompress(buf))
+
+ return size_info, fn
diff --git a/util/names.py b/util/names.py
index 2eb622737..31546d450 100644
--- a/util/names.py
+++ b/util/names.py
@@ -34,6 +34,27 @@ def parse_robot_username(robot_username):
return robot_username.split('+', 2)
+def parse_urn(urn):
+ """ Parses a URN, returning a pair that contains a list of URN
+ namespace parts, followed by the URN's unique ID.
+ """
+ if not urn.startswith('urn:'):
+ return None
+
+ parts = urn[len('urn:'):].split(':')
+ return (parts[0:len(parts) - 1], parts[len(parts) - 1])
+
+
+def parse_single_urn(urn):
+ """ Parses a URN, returning a pair that contains the first
+ namespace part, followed by the URN's unique ID.
+ """
+ result = parse_urn(urn)
+ if result is None or not len(result[0]):
+ return None
+
+ return (result[0][0], result[1])
+
uuid_generator = lambda: str(uuid4())
diff --git a/util/useremails.py b/util/useremails.py
index 0a78da7e3..ca0fd896a 100644
--- a/util/useremails.py
+++ b/util/useremails.py
@@ -1,116 +1,158 @@
from flask.ext.mail import Message
from app import mail, app, get_app_url
+from jinja2 import Template, Environment, FileSystemLoader, contextfilter
+from data import model
+from util.gravatar import compute_hash
+
+def user_reference(username):
+ user = model.get_user_or_org(username)
+ if not user:
+ return username
+
+ return """
+
+ 
+ %s
+ """ % (compute_hash(user.email), username)
-CONFIRM_MESSAGE = """
-This email address was recently used to register the username '%s'
-at Quay.io.
-
-To confirm this email address, please click the following link:
-%s/confirm?code=%s
-"""
+def repository_reference(pair):
+ (namespace, repository) = pair
+
+ owner = model.get_user(namespace)
+ if not owner:
+ return "%s/%s" % (namespace, repository)
+
+ return """
+
+
+ %s/%s
+
+ """ % (compute_hash(owner.email), get_app_url(), namespace, repository, namespace, repository)
-CHANGE_MESSAGE = """
-This email address was recently asked to become the new e-mail address for username '%s'
-at Quay.io.
-
-To confirm this email address, please click the following link:
-%s/confirm?code=%s
-"""
+def admin_reference(username):
+ user = model.get_user(username)
+ if not user:
+ return 'account settings'
+
+ if user.organization:
+ return """
+ organization's admin setting
+ """ % (get_app_url(), username)
+ else:
+ return """
+ account settings
+ """ % (get_app_url())
-RECOVERY_MESSAGE = """
-A user at Quay.io has attempted to recover their account
-using this email address.
-
-If you made this request, please click the following link to recover your account and
-change your password:
-%s/recovery?code=%s
-
-If you did not make this request, your account has not been compromised and the user was
-not given access. Please disregard this email.
-"""
+template_loader = FileSystemLoader(searchpath="emails")
+template_env = Environment(loader=template_loader)
+template_env.filters['user_reference'] = user_reference
+template_env.filters['admin_reference'] = admin_reference
+template_env.filters['repository_reference'] = repository_reference
-SUBSCRIPTION_CHANGE = """
-Change: {0}
-Customer id: {1}
-Customer email: {2}
-Quay user or org name: {3}
-"""
+def send_email(recipient, subject, template_file, parameters):
+ app_title = app.config['REGISTRY_TITLE_SHORT']
+ app_url = get_app_url()
+
+ def app_link_handler(url=None, title=None):
+ real_url = app_url + '/' + url if url else app_url
+ if not title:
+ title = real_url if url else app_title
+
+ return '%s' % (real_url, title)
+
+ parameters.update({
+ 'subject': subject,
+ 'app_logo': 'https://quay.io/static/img/quay-logo.png', # TODO: make this pull from config
+ 'app_url': app_url,
+ 'app_title': app_title,
+ 'app_link': app_link_handler
+ })
+
+ rendered_html = template_env.get_template(template_file + '.html').render(parameters)
+
+ msg = Message('[%s] %s' % (app_title, subject), sender='support@quay.io', recipients=[recipient])
+ msg.html = rendered_html
+ mail.send(msg)
-PAYMENT_FAILED = """
-Hi {0},
-
-Your recent payment for Quay.io failed, which usually results in our payments processorcanceling
-your subscription automatically. If you would like to continue to use Quay.io without interruption,
-please add a new card to Quay.io and re-subscribe to your plan.
-
-You can find the card and subscription management features under your account settings.
-
-Thanks and have a great day!
-
--Quay.io Support
-"""
-
-
-AUTH_FORREPO_MESSAGE = """
-A request has been made to send notifications to this email address for the
-Quay.io repository %s/%s.
-
-To confirm this email address, please click the following link:
-%s/authrepoemail?code=%s
-"""
-
-
-SUBSCRIPTION_CHANGE_TITLE = 'Subscription Change - {0} {1}'
+def send_password_changed(username, email):
+ send_email(email, 'Account password changed', 'passwordchanged', {
+ 'username': username
+ })
+def send_email_changed(username, old_email, new_email):
+ send_email(old_email, 'Account e-mail address changed', 'emailchanged', {
+ 'username': username,
+ 'new_email': new_email
+ })
def send_change_email(username, email, token):
- msg = Message('Quay.io email change. Please confirm your email.',
- sender='support@quay.io', # Why do I need this?
- recipients=[email])
- msg.html = CHANGE_MESSAGE % (username, get_app_url(), get_app_url(), token, get_app_url(), token)
- mail.send(msg)
-
+ send_email(email, 'E-mail address change requested', 'changeemail', {
+ 'username': username,
+ 'token': token
+ })
def send_confirmation_email(username, email, token):
- msg = Message('Welcome to Quay.io! Please confirm your email.',
- sender='support@quay.io', # Why do I need this?
- recipients=[email])
- msg.html = CONFIRM_MESSAGE % (username, get_app_url(), get_app_url(), token, get_app_url(), token)
- mail.send(msg)
-
+ send_email(email, 'Please confirm your e-mail address', 'confirmemail', {
+ 'username': username,
+ 'token': token
+ })
def send_repo_authorization_email(namespace, repository, email, token):
- msg = Message('Quay.io Notification: Please confirm your email.',
- sender='support@quay.io', # Why do I need this?
- recipients=[email])
- msg.html = AUTH_FORREPO_MESSAGE % (get_app_url(), get_app_url(), namespace, repository, namespace,
- repository, get_app_url(), token, get_app_url(), token)
- mail.send(msg)
-
+ subject = 'Please verify your e-mail address for repository %s/%s' % (namespace, repository)
+ send_email(email, subject, 'repoauthorizeemail', {
+ 'namespace': namespace,
+ 'repository': repository,
+ 'token': token
+ })
def send_recovery_email(email, token):
- msg = Message('Quay.io account recovery.',
- sender='support@quay.io', # Why do I need this?
- recipients=[email])
- msg.html = RECOVERY_MESSAGE % (get_app_url(), get_app_url(), token, get_app_url(), token)
- mail.send(msg)
+ subject = 'Account recovery'
+ send_email(email, subject, 'recovery', {
+ 'email': email,
+ 'token': token
+ })
+
+def send_payment_failed(email, username):
+ send_email(email, 'Subscription Payment Failure', 'paymentfailure', {
+ 'username': username
+ })
+
+def send_org_invite_email(member_name, member_email, orgname, team, adder, code):
+ send_email(member_email, 'Invitation to join team', 'teaminvite', {
+ 'inviter': adder,
+ 'token': code,
+ 'organization': orgname,
+ 'teamname': team
+ })
def send_invoice_email(email, contents):
+ # Note: This completely generates the contents of the email, so we don't use the
+ # normal template here.
msg = Message('Quay.io payment received - Thank you!',
- sender='support@quay.io', # Why do I need this?
+ sender='support@quay.io',
recipients=[email])
msg.html = contents
mail.send(msg)
+# INTERNAL EMAILS BELOW
+
def send_subscription_change(change_description, customer_id, customer_email, quay_username):
+ SUBSCRIPTION_CHANGE_TITLE = 'Subscription Change - {0} {1}'
+ SUBSCRIPTION_CHANGE = """
+ Change: {0}
+ Customer id: {1}
+ Customer email: {2}
+ Quay user or org name: {3}
+ """
+
title = SUBSCRIPTION_CHANGE_TITLE.format(quay_username, change_description)
msg = Message(title, sender='support@quay.io', recipients=['stripe@quay.io'])
msg.html = SUBSCRIPTION_CHANGE.format(change_description, customer_id, customer_email,
@@ -118,8 +160,3 @@ def send_subscription_change(change_description, customer_id, customer_email, qu
mail.send(msg)
-def send_payment_failed(customer_email, quay_username):
- msg = Message('Quay.io Subscription Payment Failure', sender='support@quay.io',
- recipients=[customer_email])
- msg.html = PAYMENT_FAILED.format(quay_username)
- mail.send(msg)
diff --git a/workers/buildlogsarchiver.py b/workers/buildlogsarchiver.py
index 213481eed..460fd7f4e 100644
--- a/workers/buildlogsarchiver.py
+++ b/workers/buildlogsarchiver.py
@@ -7,11 +7,12 @@ from gzip import GzipFile
from data import model
from data.archivedlogs import JSON_MIMETYPE
-from data.database import RepositoryBuild
+from data.database import RepositoryBuild, db_random_func
from app import build_logs, log_archive
from util.streamingjsonencoder import StreamingJSONEncoder
POLL_PERIOD_SECONDS = 30
+MEMORY_TEMPFILE_SIZE = 64 * 1024 # Large enough to handle approximately 99% of builds in memory
logger = logging.getLogger(__name__)
sched = BlockingScheduler()
@@ -22,7 +23,7 @@ def archive_redis_buildlogs():
avoid needing two-phase commit. """
try:
# Get a random build to archive
- to_archive = model.archivable_buildlogs_query().order_by(fn.Random()).get()
+ to_archive = model.archivable_buildlogs_query().order_by(db_random_func()).get()
logger.debug('Archiving: %s', to_archive.uuid)
length, entries = build_logs.get_log_entries(to_archive.uuid, 0)
@@ -32,7 +33,7 @@ def archive_redis_buildlogs():
'logs': entries,
}
- with SpooledTemporaryFile() as tempfile:
+ with SpooledTemporaryFile(MEMORY_TEMPFILE_SIZE) as tempfile:
with GzipFile('testarchive', fileobj=tempfile) as zipstream:
for chunk in StreamingJSONEncoder().iterencode(to_encode):
zipstream.write(chunk)
diff --git a/workers/diffsworker.py b/workers/diffsworker.py
index 70f74f1db..563c61352 100644
--- a/workers/diffsworker.py
+++ b/workers/diffsworker.py
@@ -33,7 +33,8 @@ class DiffsWorker(Worker):
return True
-logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
+if __name__ == "__main__":
+ logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
-worker = DiffsWorker(image_diff_queue)
-worker.start()
+ worker = DiffsWorker(image_diff_queue)
+ worker.start()
diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py
index e43336718..60b29a109 100644
--- a/workers/dockerfilebuild.py
+++ b/workers/dockerfilebuild.py
@@ -38,6 +38,8 @@ TIMEOUT_PERIOD_MINUTES = 20
CACHE_EXPIRATION_PERIOD_HOURS = 24
NO_TAGS = [':']
RESERVATION_TIME = (TIMEOUT_PERIOD_MINUTES + 5) * 60
+DOCKER_BASE_URL = None # Set this if you want to use a different docker URL/socket.
+
def matches_system_error(status_str):
""" Returns true if the given status string matches a known system error in the
@@ -128,8 +130,8 @@ class DockerfileBuildContext(object):
# Note: We have two different clients here because we (potentially) login
# with both, but with different credentials that we do not want shared between
# the build and push operations.
- self._push_cl = StreamingDockerClient(timeout=1200)
- self._build_cl = StreamingDockerClient(timeout=1200)
+ self._push_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL)
+ self._build_cl = StreamingDockerClient(timeout=1200, base_url = DOCKER_BASE_URL)
dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
'Dockerfile')
@@ -221,6 +223,13 @@ class DockerfileBuildContext(object):
raise RuntimeError(message)
def pull(self):
+ image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag()
+ if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
+ self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR)
+ raise JobException('Missing FROM command in Dockerfile')
+
+ image_and_tag = ':'.join(image_and_tag_tuple)
+
# Login with the specified credentials (if any).
if self._pull_credentials:
logger.debug('Logging in with pull credentials: %s@%s',
@@ -236,13 +245,6 @@ class DockerfileBuildContext(object):
registry=self._pull_credentials['registry'], reauth=True)
# Pull the image, in case it was updated since the last build
- image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag()
- if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
- self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR)
- raise JobException('Missing FROM command in Dockerfile')
-
- image_and_tag = ':'.join(image_and_tag_tuple)
-
self._build_logger('Pulling base image: %s' % image_and_tag, log_data = {
'phasestep': 'pull',
'repo_url': image_and_tag
@@ -478,9 +480,8 @@ class DockerfileBuildWorker(Worker):
def watchdog(self):
logger.debug('Running build watchdog code.')
-
try:
- docker_cl = Client()
+ docker_cl = Client(base_url = DOCKER_BASE_URL)
# Iterate the running containers and kill ones that have been running more than 20 minutes
for container in docker_cl.containers():
@@ -519,7 +520,20 @@ class DockerfileBuildWorker(Worker):
log_appender = partial(build_logs.append_log_message,
repository_build.uuid)
- log_appender('initializing', build_logs.PHASE)
+ # Lookup and save the version of docker being used.
+ docker_cl = Client(base_url = DOCKER_BASE_URL)
+ docker_version = docker_cl.version().get('Version', '')
+ dash = docker_version.find('-')
+
+ # Strip any -tutum or whatever off of the version.
+ if dash > 0:
+ docker_version = docker_version[:dash]
+
+ log_appender('initializing', build_logs.PHASE, log_data = {
+ 'docker_version': docker_version
+ })
+
+ log_appender('Docker version: %s' % docker_version)
start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url,
repo))
diff --git a/workers/notificationworker.py b/workers/notificationworker.py
index 2af0954f9..a176d46c8 100644
--- a/workers/notificationworker.py
+++ b/workers/notificationworker.py
@@ -8,6 +8,7 @@ from workers.worker import Worker
from endpoints.notificationmethod import NotificationMethod, InvalidNotificationMethodException
from endpoints.notificationevent import NotificationEvent, InvalidNotificationEventException
+from workers.worker import JobException
from data import model
@@ -29,7 +30,7 @@ class NotificationWorker(Worker):
notification = model.get_repo_notification(repo_namespace, repo_name, notification_uuid)
if not notification:
# Probably deleted.
- return True
+ return
event_name = notification.event.name
method_name = notification.method.name
@@ -39,15 +40,17 @@ class NotificationWorker(Worker):
method_handler = NotificationMethod.get_method(method_name)
except InvalidNotificationMethodException as ex:
logger.exception('Cannot find notification method: %s' % ex.message)
- return False
+ raise JobException('Cannot find notification method: %s' % ex.message)
except InvalidNotificationEventException as ex:
- logger.exception('Cannot find notification method: %s' % ex.message)
- return False
+ logger.exception('Cannot find notification event: %s' % ex.message)
+ raise JobException('Cannot find notification event: %s' % ex.message)
- return method_handler.perform(notification, event_handler, job_details)
+ method_handler.perform(notification, event_handler, job_details)
-logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
-worker = NotificationWorker(notification_queue, poll_period_seconds=15,
- reservation_seconds=3600)
-worker.start()
+if __name__ == "__main__":
+ logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
+
+ worker = NotificationWorker(notification_queue, poll_period_seconds=10, reservation_seconds=30,
+ retry_after_seconds=30)
+ worker.start()
diff --git a/workers/worker.py b/workers/worker.py
index 57d4a02d0..03066dc6e 100644
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -63,11 +63,12 @@ class WorkerStatusHandler(BaseHTTPRequestHandler):
class Worker(object):
def __init__(self, queue, poll_period_seconds=30, reservation_seconds=300,
- watchdog_period_seconds=60):
+ watchdog_period_seconds=60, retry_after_seconds=300):
self._sched = BackgroundScheduler()
self._poll_period_seconds = poll_period_seconds
self._reservation_seconds = reservation_seconds
self._watchdog_period_seconds = watchdog_period_seconds
+ self._retry_after_seconds = retry_after_seconds
self._stop = Event()
self._terminated = Event()
self._queue = queue
@@ -103,7 +104,8 @@ class Worker(object):
try:
self.watchdog()
except WorkerUnhealthyException as exc:
- logger.error('The worker has encountered an error via watchdog and will not take new jobs: %s' % exc.message)
+ logger.error('The worker has encountered an error via watchdog and will not take new jobs')
+ logger.error(exc.message)
self.mark_current_incomplete(restore_retry=True)
self._stop.set()
@@ -111,7 +113,7 @@ class Worker(object):
logger.debug('Getting work item from queue.')
with self._current_item_lock:
- self.current_queue_item = self._queue.get()
+ self.current_queue_item = self._queue.get(processing_time=self._reservation_seconds)
while True:
# Retrieve the current item in the queue over which to operate. We do so under
@@ -129,12 +131,14 @@ class Worker(object):
self.process_queue_item(job_details)
self.mark_current_complete()
- except JobException:
+ except JobException as jex:
logger.warning('An error occurred processing request: %s', current_queue_item.body)
+ logger.warning('Job exception: %s' % jex)
self.mark_current_incomplete(restore_retry=False)
except WorkerUnhealthyException as exc:
- logger.error('The worker has encountered an error via the job and will not take new jobs: %s' % exc.message)
+ logger.error('The worker has encountered an error via the job and will not take new jobs')
+ logger.error(exc.message)
self.mark_current_incomplete(restore_retry=True)
self._stop.set()
@@ -190,7 +194,8 @@ class Worker(object):
def mark_current_incomplete(self, restore_retry=False):
with self._current_item_lock:
if self.current_queue_item is not None:
- self._queue.incomplete(self.current_queue_item, restore_retry=restore_retry)
+ self._queue.incomplete(self.current_queue_item, restore_retry=restore_retry,
+ retry_after=self._retry_after_seconds)
self.current_queue_item = None
def mark_current_complete(self):
+