Merge remote-tracking branch 'origin/master' into comewithmeifyouwanttowork

Conflicts:
	data/model/legacy.py
	static/js/app.js
This commit is contained in:
Jake Moshenko 2014-09-12 11:03:30 -04:00
commit c5ca46a14b
70 changed files with 1566 additions and 630 deletions

View file

@ -1,7 +1,8 @@
import logging
import json
import datetime
from flask import Blueprint, request, make_response, jsonify
from flask import Blueprint, request, make_response, jsonify, session
from flask.ext.restful import Resource, abort, Api, reqparse
from flask.ext.restful.utils.cors import crossdomain
from werkzeug.exceptions import HTTPException
@ -66,6 +67,11 @@ class Unauthorized(ApiException):
ApiException.__init__(self, 'insufficient_scope', 403, 'Unauthorized', payload)
class FreshLoginRequired(ApiException):
def __init__(self, payload=None):
ApiException.__init__(self, 'fresh_login_required', 401, "Requires fresh login", payload)
class ExceedsLicenseException(ApiException):
def __init__(self, payload=None):
ApiException.__init__(self, None, 402, 'Payment Required', payload)
@ -87,6 +93,14 @@ def handle_api_error(error):
return response
@api_bp.app_errorhandler(model.TooManyLoginAttemptsException)
@crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
def handle_too_many_login_attempts(error):
response = make_response('Too many login attempts', 429)
response.headers['Retry-After'] = int(error.retry_after)
return response
def resource(*urls, **kwargs):
def wrapper(api_resource):
if not api_resource:
@ -256,6 +270,26 @@ def require_user_permission(permission_class, scope=None):
require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER)
require_user_admin = require_user_permission(UserAdminPermission, None)
require_fresh_user_admin = require_user_permission(UserAdminPermission, None)
def require_fresh_login(func):
@add_method_metadata('requires_fresh_login', True)
@wraps(func)
def wrapped(*args, **kwargs):
user = get_authenticated_user()
if not user:
raise Unauthorized()
logger.debug('Checking fresh login for user %s', user.username)
last_login = session.get('login_time', datetime.datetime.min)
valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10)
if not user.password_hash or last_login >= valid_span:
return func(*args, **kwargs)
raise FreshLoginRequired()
return wrapped
def require_scope(scope_object):

View file

@ -80,7 +80,7 @@ def build_status_view(build_obj, can_write=False):
}
if can_write:
resp['archive_url'] = user_files.get_file_url(build_obj.resource_key)
resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
return resp
@ -257,7 +257,7 @@ class FileDropResource(ApiResource):
def post(self):
""" Request a URL to which a file may be uploaded. """
mime_type = request.get_json()['mimeType']
(url, file_id) = user_files.prepare_for_drop(mime_type)
(url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True)
return {
'url': url,
'file_id': str(file_id),

View file

@ -119,6 +119,11 @@ def swagger_route_data(include_internal=False, compact=False):
if internal is not None:
new_operation['internal'] = True
if include_internal:
requires_fresh_login = method_metadata(method, 'requires_fresh_login')
if requires_fresh_login is not None:
new_operation['requires_fresh_login'] = True
if not internal or (internal and include_internal):
operations.append(new_operation)

View file

@ -9,7 +9,7 @@ from app import app, billing as stripe, authentication
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
log_action, internal_only, NotFound, require_user_admin, parse_args,
query_param, InvalidToken, require_scope, format_date, hide_if, show_if,
license_error)
license_error, require_fresh_login)
from endpoints.api.subscribe import subscribe
from endpoints.common import common_login
from endpoints.api.team import try_accept_invite
@ -43,9 +43,15 @@ def user_view(user):
organizations = model.get_user_organizations(user.username)
def login_view(login):
try:
metadata = json.loads(login.metadata_json)
except:
metadata = {}
return {
'service': login.service.name,
'service_identifier': login.service_ident,
'metadata': metadata
}
logins = model.list_federated_logins(user)
@ -92,6 +98,7 @@ class User(ApiResource):
""" Operations related to users. """
schemas = {
'NewUser': {
'id': 'NewUser',
'type': 'object',
'description': 'Fields which must be specified for a new user.',
@ -147,6 +154,7 @@ class User(ApiResource):
return user_view(user)
@require_user_admin
@require_fresh_login
@nickname('changeUserDetails')
@internal_only
@validate_json_request('UpdateUser')
@ -155,7 +163,7 @@ class User(ApiResource):
user = get_authenticated_user()
user_data = request.get_json()
try:
try:
if 'password' in user_data:
logger.debug('Changing password for user: %s', user.username)
log_action('account_change_password', user.username)
@ -372,6 +380,37 @@ class Signin(ApiResource):
return conduct_signin(username, password)
@resource('/v1/signin/verify')
@internal_only
class VerifyUser(ApiResource):
""" Operations for verifying the existing user. """
schemas = {
'VerifyUser': {
'id': 'VerifyUser',
'type': 'object',
'description': 'Information required to verify the signed in user.',
'required': [
'password',
],
'properties': {
'password': {
'type': 'string',
'description': 'The user\'s password',
},
},
},
}
@require_user_admin
@nickname('verifyUser')
@validate_json_request('VerifyUser')
def post(self):
""" Verifies the signed in the user with the specified credentials. """
signin_data = request.get_json()
password = signin_data['password']
return conduct_signin(get_authenticated_user().username, password)
@resource('/v1/signout')
@internal_only
class Signout(ApiResource):

View file

@ -4,12 +4,14 @@ from flask import request, redirect, url_for, Blueprint
from flask.ext.login import current_user
from endpoints.common import render_page_template, common_login, route_show_if
from app import app, analytics
from app import app, analytics, get_app_url
from data import model
from util.names import parse_repository_name
from util.validation import generate_valid_usernames
from util.http import abort
from auth.permissions import AdministerRepositoryPermission
from auth.auth import require_session_login
from peewee import IntegrityError
import features
@ -20,20 +22,39 @@ client = app.config['HTTPCLIENT']
callback = Blueprint('callback', __name__)
def render_ologin_error(service_name,
error_message='Could not load user data. The token may have expired.'):
return render_page_template('ologinerror.html', service_name=service_name,
error_message=error_message,
service_url=get_app_url())
def exchange_github_code_for_token(code, for_login=True):
def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False,
redirect_suffix=''):
code = request.args.get('code')
id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID'
secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET'
payload = {
'client_id': app.config['GITHUB_LOGIN_CLIENT_ID' if for_login else 'GITHUB_CLIENT_ID'],
'client_secret': app.config['GITHUB_LOGIN_CLIENT_SECRET' if for_login else 'GITHUB_CLIENT_SECRET'],
'client_id': app.config[id_config],
'client_secret': app.config[secret_config],
'code': code,
'grant_type': 'authorization_code',
'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
service_name.lower(),
redirect_suffix)
}
headers = {
'Accept': 'application/json'
}
get_access_token = client.post(app.config['GITHUB_TOKEN_URL'],
params=payload, headers=headers)
if form_encode:
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
data=payload, headers=headers)
else:
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
params=payload, headers=headers)
json_data = get_access_token.json()
if not json_data:
@ -52,17 +73,82 @@ def get_github_user(token):
return get_user.json()
def get_google_user(token):
token_param = {
'access_token': token,
'alt': 'json',
}
get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param)
return get_user.json()
def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
to_login = model.verify_federated_login(service_name.lower(), user_id)
if not to_login:
# try to create the user
try:
valid = next(generate_valid_usernames(username))
to_login = model.create_federated_user(valid, email, service_name.lower(),
user_id, set_password_notification=True,
metadata=metadata)
# Success, tell analytics
analytics.track(to_login.username, 'register', {'service': service_name.lower()})
state = request.args.get('state', None)
if state:
logger.debug('Aliasing with state: %s' % state)
analytics.alias(to_login.username, state)
except model.DataModelException, ex:
return render_ologin_error(service_name, ex.message)
if common_login(to_login):
return redirect(url_for('web.index'))
return render_ologin_error(service_name)
def get_google_username(user_data):
username = user_data['email']
at = username.find('@')
if at > 0:
username = username[0:at]
return username
@callback.route('/google/callback', methods=['GET'])
@route_show_if(features.GOOGLE_LOGIN)
def google_oauth_callback():
error = request.args.get('error', None)
if error:
return render_ologin_error('Google', error)
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True)
user_data = get_google_user(token)
if not user_data or not user_data.get('id', None) or not user_data.get('email', None):
return render_ologin_error('Google')
username = get_google_username(user_data)
metadata = {
'service_username': user_data['email']
}
return conduct_oauth_login('Google', user_data['id'], username, user_data['email'],
metadata=metadata)
@callback.route('/github/callback', methods=['GET'])
@route_show_if(features.GITHUB_LOGIN)
def github_oauth_callback():
error = request.args.get('error', None)
if error:
return render_page_template('githuberror.html', error_message=error)
return render_ologin_error('GitHub', error)
token = exchange_github_code_for_token(request.args.get('code'))
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
user_data = get_github_user(token)
if not user_data:
return render_page_template('githuberror.html', error_message='Could not load user data')
return render_ologin_error('GitHub')
username = user_data['login']
github_id = user_data['id']
@ -84,42 +170,67 @@ def github_oauth_callback():
if user_email['primary']:
break
to_login = model.verify_federated_login('github', github_id)
if not to_login:
# try to create the user
try:
to_login = model.create_federated_user(username, found_email, 'github',
github_id, set_password_notification=True)
metadata = {
'service_username': username
}
# Success, tell analytics
analytics.track(to_login.username, 'register', {'service': 'github'})
return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata)
state = request.args.get('state', None)
if state:
logger.debug('Aliasing with state: %s' % state)
analytics.alias(to_login.username, state)
except model.DataModelException, ex:
return render_page_template('githuberror.html', error_message=ex.message)
@callback.route('/google/callback/attach', methods=['GET'])
@route_show_if(features.GOOGLE_LOGIN)
@require_session_login
def google_oauth_attach():
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE',
redirect_suffix='/attach', form_encode=True)
if common_login(to_login):
return redirect(url_for('web.index'))
user_data = get_google_user(token)
if not user_data or not user_data.get('id', None):
return render_ologin_error('Google')
return render_page_template('githuberror.html')
google_id = user_data['id']
user_obj = current_user.db_user()
username = get_google_username(user_data)
metadata = {
'service_username': user_data['email']
}
try:
model.attach_federated_login(user_obj, 'google', google_id, metadata=metadata)
except IntegrityError:
err = 'Google account %s is already attached to a %s account' % (
username, app.config['REGISTRY_TITLE_SHORT'])
return render_ologin_error('Google', err)
return redirect(url_for('web.user'))
@callback.route('/github/callback/attach', methods=['GET'])
@route_show_if(features.GITHUB_LOGIN)
@require_session_login
def github_oauth_attach():
token = exchange_github_code_for_token(request.args.get('code'))
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
user_data = get_github_user(token)
if not user_data:
return render_page_template('githuberror.html', error_message='Could not load user data')
return render_ologin_error('GitHub')
github_id = user_data['id']
user_obj = current_user.db_user()
model.attach_federated_login(user_obj, 'github', github_id)
username = user_data['login']
metadata = {
'service_username': username
}
try:
model.attach_federated_login(user_obj, 'github', github_id, metadata=metadata)
except IntegrityError:
err = 'Github account %s is already attached to a %s account' % (
username, app.config['REGISTRY_TITLE_SHORT'])
return render_ologin_error('GitHub', err)
return redirect(url_for('web.user'))
@ -130,7 +241,8 @@ def github_oauth_attach():
def attach_github_build_trigger(namespace, repository):
permission = AdministerRepositoryPermission(namespace, repository)
if permission.can():
token = exchange_github_code_for_token(request.args.get('code'), for_login=False)
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB',
for_login=False)
repo = model.get_repository(namespace, repository)
if not repo:
msg = 'Invalid repository: %s/%s' % (namespace, repository)

View file

@ -2,8 +2,9 @@ import logging
import urlparse
import json
import string
import datetime
from flask import make_response, render_template, request, abort
from flask import make_response, render_template, request, abort, session
from flask.ext.login import login_user, UserMixin
from flask.ext.principal import identity_changed
from random import SystemRandom
@ -112,6 +113,7 @@ def common_login(db_user):
logger.debug('Successfully signed in as: %s' % db_user.username)
new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN})
identity_changed.send(app, identity=new_identity)
session['login_time'] = datetime.datetime.now()
return True
else:
logger.debug('User could not be logged in, inactive?.')

View file

@ -15,6 +15,13 @@ class NotificationEvent(object):
def __init__(self):
pass
def get_level(self, event_data, notification_data):
"""
Returns a 'level' representing the severity of the event.
Valid values are: 'info', 'warning', 'error', 'primary'
"""
raise NotImplementedError
def get_summary(self, event_data, notification_data):
"""
Returns a human readable one-line summary for the given notification data.
@ -55,6 +62,9 @@ class RepoPushEvent(NotificationEvent):
def event_name(cls):
return 'repo_push'
def get_level(self, event_data, notification_data):
return 'info'
def get_summary(self, event_data, notification_data):
return 'Repository %s updated' % (event_data['repository'])
@ -87,6 +97,9 @@ class BuildQueueEvent(NotificationEvent):
@classmethod
def event_name(cls):
return 'build_queued'
def get_level(self, event_data, notification_data):
return 'info'
def get_sample_data(self, repository):
build_uuid = 'fake-build-id'
@ -127,6 +140,9 @@ class BuildStartEvent(NotificationEvent):
def event_name(cls):
return 'build_start'
def get_level(self, event_data, notification_data):
return 'info'
def get_sample_data(self, repository):
build_uuid = 'fake-build-id'
@ -155,6 +171,9 @@ class BuildSuccessEvent(NotificationEvent):
def event_name(cls):
return 'build_success'
def get_level(self, event_data, notification_data):
return 'primary'
def get_sample_data(self, repository):
build_uuid = 'fake-build-id'
@ -183,6 +202,9 @@ class BuildFailureEvent(NotificationEvent):
def event_name(cls):
return 'build_failure'
def get_level(self, event_data, notification_data):
return 'error'
def get_sample_data(self, repository):
build_uuid = 'fake-build-id'

View file

@ -4,9 +4,11 @@ import os.path
import tarfile
import base64
import json
import requests
import re
from flask.ext.mail import Message
from app import mail, app
from app import mail, app, get_app_url
from data import model
logger = logging.getLogger(__name__)
@ -187,3 +189,194 @@ class WebhookMethod(NotificationMethod):
return False
return True
class FlowdockMethod(NotificationMethod):
""" Method for sending notifications to Flowdock via the Team Inbox API:
https://www.flowdock.com/api/team-inbox
"""
@classmethod
def method_name(cls):
return 'flowdock'
def validate(self, repository, config_data):
token = config_data.get('flow_api_token', '')
if not token:
raise CannotValidateNotificationMethodException('Missing Flowdock API Token')
def perform(self, notification, event_handler, notification_data):
config_data = json.loads(notification.config_json)
token = config_data.get('flow_api_token', '')
if not token:
return False
owner = model.get_user(notification.repository.namespace)
if not owner:
# Something went wrong.
return False
url = 'https://api.flowdock.com/v1/messages/team_inbox/%s' % token
headers = {'Content-type': 'application/json'}
payload = {
'source': 'Quay',
'from_address': 'support@quay.io',
'subject': event_handler.get_summary(notification_data['event_data'], notification_data),
'content': event_handler.get_message(notification_data['event_data'], notification_data),
'from_name': owner.username,
'project': notification.repository.namespace + ' ' + notification.repository.name,
'tags': ['#' + event_handler.event_name()],
'link': notification_data['event_data']['homepage']
}
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
logger.error('%s response for flowdock to url: %s' % (resp.status_code,
url))
logger.error(resp.content)
return False
except requests.exceptions.RequestException as ex:
logger.exception('Flowdock method was unable to be sent: %s' % ex.message)
return False
return True
class HipchatMethod(NotificationMethod):
""" Method for sending notifications to Hipchat via the API:
https://www.hipchat.com/docs/apiv2/method/send_room_notification
"""
@classmethod
def method_name(cls):
return 'hipchat'
def validate(self, repository, config_data):
if not config_data.get('notification_token', ''):
raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token')
if not config_data.get('room_id', ''):
raise CannotValidateNotificationMethodException('Missing Hipchat Room ID')
def perform(self, notification, event_handler, notification_data):
config_data = json.loads(notification.config_json)
token = config_data.get('notification_token', '')
room_id = config_data.get('room_id', '')
if not token or not room_id:
return False
owner = model.get_user(notification.repository.namespace)
if not owner:
# Something went wrong.
return False
url = 'https://api.hipchat.com/v2/room/%s/notification?auth_token=%s' % (room_id, token)
level = event_handler.get_level(notification_data['event_data'], notification_data)
color = {
'info': 'gray',
'warning': 'yellow',
'error': 'red',
'primary': 'purple'
}.get(level, 'gray')
headers = {'Content-type': 'application/json'}
payload = {
'color': color,
'message': event_handler.get_message(notification_data['event_data'], notification_data),
'notify': level == 'error',
'message_format': 'html',
}
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
logger.error('%s response for hipchat to url: %s' % (resp.status_code,
url))
logger.error(resp.content)
return False
except requests.exceptions.RequestException as ex:
logger.exception('Hipchat method was unable to be sent: %s' % ex.message)
return False
return True
class SlackMethod(NotificationMethod):
""" Method for sending notifications to Slack via the API:
https://api.slack.com/docs/attachments
"""
@classmethod
def method_name(cls):
return 'slack'
def validate(self, repository, config_data):
if not config_data.get('token', ''):
raise CannotValidateNotificationMethodException('Missing Slack Token')
if not config_data.get('subdomain', '').isalnum():
raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name')
def formatForSlack(self, message):
message = message.replace('\n', '')
message = re.sub(r'\s+', ' ', message)
message = message.replace('<br>', '\n')
message = re.sub(r'<a href="(.+)">(.+)</a>', '<\\1|\\2>', message)
return message
def perform(self, notification, event_handler, notification_data):
config_data = json.loads(notification.config_json)
token = config_data.get('token', '')
subdomain = config_data.get('subdomain', '')
if not token or not subdomain:
return False
owner = model.get_user(notification.repository.namespace)
if not owner:
# Something went wrong.
return False
url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
level = event_handler.get_level(notification_data['event_data'], notification_data)
color = {
'info': '#ffffff',
'warning': 'warning',
'error': 'danger',
'primary': 'good'
}.get(level, '#ffffff')
summary = event_handler.get_summary(notification_data['event_data'], notification_data)
message = event_handler.get_message(notification_data['event_data'], notification_data)
headers = {'Content-type': 'application/json'}
payload = {
'text': summary,
'username': 'quayiobot',
'attachments': [
{
'fallback': summary,
'text': self.formatForSlack(message),
'color': color
}
]
}
try:
resp = requests.post(url, data=json.dumps(payload), headers=headers)
if resp.status_code/100 != 2:
logger.error('%s response for Slack to url: %s' % (resp.status_code,
url))
logger.error(resp.content)
return False
except requests.exceptions.RequestException as ex:
logger.exception('Slack method was unable to be sent: %s' % ex.message)
return False
return True

View file

@ -110,10 +110,10 @@ def head_image_layer(namespace, repository, image_id, headers):
extra_headers = {}
# Add the Accept-Ranges header if the storage engine supports resumeable
# Add the Accept-Ranges header if the storage engine supports resumable
# downloads.
if store.get_supports_resumeable_downloads(repo_image.storage.locations):
profile.debug('Storage supports resumeable downloads')
if store.get_supports_resumable_downloads(repo_image.storage.locations):
profile.debug('Storage supports resumable downloads')
extra_headers['Accept-Ranges'] = 'bytes'
resp = make_response('')

View file

@ -291,6 +291,9 @@ class GithubBuildTrigger(BuildTrigger):
with tarfile.open(fileobj=tarball) as archive:
tarball_subdir = archive.getnames()[0]
# Seek to position 0 to make boto multipart happy
tarball.seek(0)
dockerfile_id = user_files.store_file(tarball, TARBALL_MIME)
logger.debug('Successfully prepared job')