import json import logging from urlparse import urlparse from datetime import timedelta from cachetools import lru_cache from flask import (abort, redirect, request, url_for, make_response, Response, render_template, Blueprint, send_from_directory, jsonify, send_file) from flask.ext.login import current_user import features from app import (app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider, get_app_url, instance_keys) from auth import scopes from auth.auth import require_session_login, process_oauth, has_basic_auth, process_auth_or_cookie from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission, SuperUserPermission, AdministerRepositoryPermission, ModifyRepositoryPermission) from auth.auth_context import get_authenticated_user from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.bitbuckethandler import BitbucketBuildTrigger from buildtrigger.customhandler import CustomBuildTrigger from buildtrigger.triggerutil import TriggerProviderException from data import model from data.database import db from endpoints.api.discovery import swagger_route_data from endpoints.common import (common_login, render_page_template, route_show_if, param_required, parse_repository_name) from endpoints.csrf import csrf_protect, generate_csrf_token, verify_csrf from endpoints.decorators import anon_protect, anon_allowed from health.healthcheck import get_healthchecker from util.cache import no_cache from util.headers import parse_basic_auth from util.invoice import renderInvoiceToPdf from util.systemlogs import build_logs_archive from util.useremails import send_email_changed @lru_cache(maxsize=1) def _get_route_data(): return swagger_route_data(include_internal=True, compact=True) def render_page_template_with_routedata(name, *args, **kwargs): return render_page_template(name, _get_route_data(), *args, **kwargs) # Capture the unverified SSL errors. logger = logging.getLogger(__name__) logging.captureWarnings(True) web = Blueprint('web', __name__) STATUS_TAGS = app.config['STATUS_TAGS'] @web.route('/', methods=['GET'], defaults={'path': ''}) @no_cache def index(path, **kwargs): return render_page_template_with_routedata('index.html', **kwargs) @web.route('/500', methods=['GET']) def internal_error_display(): return render_page_template_with_routedata('500.html') @web.errorhandler(404) @web.route('/404', methods=['GET']) def not_found_error_display(e = None): resp = render_page_template_with_routedata('404.html') resp.status_code = 404 return resp @web.route('/organization/', methods=['GET']) @no_cache def org_view(path): return index('') @web.route('/user/', methods=['GET']) @no_cache def user_view(path): return index('') @route_show_if(features.ACI_CONVERSION) @web.route('/aci-signing-key') @no_cache @anon_protect def aci_signing_key(): if not signer.name: abort(404) return send_file(signer.open_public_key_file()) @web.route('/plans/') @no_cache @route_show_if(features.BILLING) def plans(): return index('') @web.route('/guide/') @no_cache def guide(): return index('') @web.route('/tour/') @web.route('/tour/') @no_cache def tour(path = ''): return index('') @web.route('/tutorial/') @no_cache def tutorial(): return index('') @web.route('/organizations/') @web.route('/organizations/new/') @no_cache def organizations(): return index('') @web.route('/user/') @no_cache def user(): return index('') @web.route('/superuser/') @no_cache @route_show_if(features.SUPER_USERS) def superuser(): return index('') @web.route('/setup/') @no_cache @route_show_if(features.SUPER_USERS) def setup(): return index('') @web.route('/signin/') @no_cache def signin(redirect=None): return index('') @web.route('/contact/') @no_cache def contact(): return index('') @web.route('/about/') @no_cache def about(): return index('') @web.route('/new/') @no_cache def new(): return index('') @web.route('/confirminvite') @no_cache def confirm_invite(): code = request.values['code'] return index('', code=code) @web.route('/repository/', defaults={'path': ''}) @web.route('/repository/', methods=['GET']) @no_cache def repository(path): return index('') @web.route('/starred/') @no_cache def starred(): return index('') @web.route('/security/') @no_cache def security(): return index('') @web.route('/enterprise/') @no_cache @route_show_if(features.BILLING) def enterprise(): return index('') @web.route('/__exp/') @no_cache def exp(expname): return index('') @web.route('/v1') @web.route('/v1/') @no_cache def v1(): return index('') @web.route('/tos', methods=['GET']) @no_cache def tos(): return index('') @web.route('/privacy', methods=['GET']) @no_cache def privacy(): return index('') # TODO(jschorr): Remove this mirrored endpoint once we migrate ELB. @web.route('/health', methods=['GET']) @web.route('/health/instance', methods=['GET']) @no_cache def instance_health(): checker = get_healthchecker(app, config_provider, instance_keys) (data, status_code) = checker.check_instance() response = jsonify(dict(data=data, status_code=status_code)) response.status_code = status_code return response # TODO(jschorr): Remove this mirrored endpoint once we migrate pingdom. @web.route('/status', methods=['GET']) @web.route('/health/endtoend', methods=['GET']) @no_cache def endtoend_health(): checker = get_healthchecker(app, config_provider, instance_keys) (data, status_code) = checker.check_endtoend() response = jsonify(dict(data=data, status_code=status_code)) response.status_code = status_code return response @web.route('/health/dbrevision', methods=['GET']) @route_show_if(features.BILLING) # Since this is only used in production. @no_cache def dbrevision_health(): # Find the revision from the database. result = db.execute_sql('select * from alembic_version limit 1').fetchone() db_revision = result[0] # Find the local revision from the file system. with open('ALEMBIC_HEAD', 'r') as f: local_revision = f.readline().split(' ')[0] data = { 'db_revision': db_revision, 'local_revision': local_revision, } status_code = 200 if db_revision == local_revision else 400 response = jsonify(dict(data=data, status_code=status_code)) response.status_code = status_code return response @web.route('/disclaimer', methods=['GET']) @no_cache def disclaimer(): return render_page_template_with_routedata('disclaimer.html') @web.route('/robots.txt', methods=['GET']) def robots(): robots_txt = make_response(render_template('robots.txt', baseurl=get_app_url())) robots_txt.headers['Content-Type'] = 'text/plain' return robots_txt @web.route('/sitemap.xml', methods=['GET']) def sitemap(): popular_repo_tuples = model.repository.list_popular_public_repos(50, timedelta(weeks=1)) xml = make_response(render_template('sitemap.xml', public_repos=popular_repo_tuples, baseurl=get_app_url())) xml.headers['Content-Type'] = 'application/xml' return xml @web.route('/buildlogs/', methods=['GET']) @route_show_if(features.BUILD_SUPPORT) @require_session_login def buildlogs(build_uuid): found_build = model.build.get_repository_build(build_uuid) if not found_build: abort(403) repo = found_build.repository if not ModifyRepositoryPermission(repo.namespace_user.username, repo.name).can(): abort(403) # If the logs have been archived, just return a URL of the completed archive if found_build.logs_archived: return redirect(log_archive.get_file_url(found_build.uuid)) _, logs = build_logs.get_log_entries(found_build.uuid, 0) response = jsonify({ 'logs': [log for log in logs] }) response.headers["Content-Disposition"] = "attachment;filename=" + found_build.uuid + ".json" return response @web.route('/receipt', methods=['GET']) @route_show_if(features.BILLING) @require_session_login def receipt(): if not current_user.is_authenticated: abort(401) return invoice_id = request.args.get('id') if invoice_id: invoice = stripe.Invoice.retrieve(invoice_id) if invoice: user_or_org = model.user.get_user_or_org_by_customer_id(invoice.customer) if user_or_org: if user_or_org.organization: admin_org = AdministerOrganizationPermission(user_or_org.username) if not admin_org.can(): abort(404) return else: if not user_or_org.username == current_user.db_user().username: abort(404) return file_data = renderInvoiceToPdf(invoice, user_or_org) return Response(file_data, mimetype="application/pdf", headers={"Content-Disposition": "attachment;filename=receipt.pdf"}) abort(404) @web.route('/authrepoemail', methods=['GET']) @route_show_if(features.MAILING) def confirm_repo_email(): code = request.values['code'] record = None try: record = model.repository.confirm_email_authorization_for_repo(code) except model.DataModelException as ex: return render_page_template_with_routedata('confirmerror.html', error_message=ex.message) message = """ Your E-mail address has been authorized to receive notifications for repository %s/%s. """ % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], record.repository.namespace_user.username, record.repository.name, record.repository.namespace_user.username, record.repository.name) return render_page_template_with_routedata('message.html', message=message) @web.route('/confirm', methods=['GET']) @route_show_if(features.MAILING) @anon_allowed def confirm_email(): code = request.values['code'] user = None new_email = None try: user, new_email, old_email = model.user.confirm_user_email(code) except model.DataModelException as ex: return render_page_template_with_routedata('confirmerror.html', error_message=ex.message) if new_email: send_email_changed(user.username, old_email, new_email) common_login(user) return redirect(url_for('web.user', tab='email') if new_email else url_for('web.index')) @web.route('/recovery', methods=['GET']) @route_show_if(features.MAILING) @anon_allowed def confirm_recovery(): code = request.values['code'] user = model.user.validate_reset_code(code) if user is not None: common_login(user) return redirect(url_for('web.user')) else: message = 'Invalid recovery code: This code is invalid or may have already been used.' return render_page_template_with_routedata('message.html', message=message) @web.route('/repository//status', methods=['GET']) @parse_repository_name() @anon_protect def build_status_badge(namespace_name, repo_name): token = request.args.get('token', None) is_public = model.repository.repository_is_public(namespace_name, repo_name) if not is_public: repo = model.repository.get_repository(namespace_name, repo_name) if not repo or token != repo.badge_token: abort(404) # Lookup the tags for the repository. tags = model.tag.list_repository_tags(namespace_name, repo_name) is_empty = len(list(tags)) == 0 recent_build = model.build.get_recent_repository_build(namespace_name, repo_name) if not is_empty and (not recent_build or recent_build.phase == 'complete'): status_name = 'ready' elif recent_build and recent_build.phase == 'error': status_name = 'failed' elif recent_build and recent_build.phase != 'complete': status_name = 'building' else: status_name = 'none' if request.headers.get('If-None-Match') == status_name: return Response(status=304) response = make_response(STATUS_TAGS[status_name]) response.content_type = 'image/svg+xml' response.headers['Cache-Control'] = 'no-cache' response.headers['ETag'] = status_name return response class FlaskAuthorizationProvider(model.oauth.DatabaseAuthorizationProvider): def get_authorized_user(self): return get_authenticated_user() def _make_response(self, body='', headers=None, status_code=200): return make_response(body, status_code, headers) @web.route('/oauth/authorizeapp', methods=['POST']) @process_auth_or_cookie def authorize_application(): # Check for an authenticated user. if not get_authenticated_user(): abort(401) return # If direct OAuth is not enabled or the user is not directly authed, verify CSRF. client_id = request.form.get('client_id', None) whitelist = app.config.get('DIRECT_OAUTH_CLIENTID_WHITELIST', []) if client_id not in whitelist or not has_basic_auth(get_authenticated_user().username): verify_csrf() provider = FlaskAuthorizationProvider() redirect_uri = request.form.get('redirect_uri', None) scope = request.form.get('scope', None) # Add the access token. return provider.get_token_response('token', client_id, redirect_uri, scope=scope) @web.route(app.config['LOCAL_OAUTH_HANDLER'], methods=['GET']) def oauth_local_handler(): if not current_user.is_authenticated: abort(401) return if not request.args.get('scope'): return render_page_template_with_routedata("message.html", message="Authorization canceled") else: return render_page_template_with_routedata("generatedtoken.html") @web.route('/oauth/denyapp', methods=['POST']) @csrf_protect def deny_application(): if not current_user.is_authenticated: abort(401) return provider = FlaskAuthorizationProvider() client_id = request.form.get('client_id', None) redirect_uri = request.form.get('redirect_uri', None) scope = request.form.get('scope', None) # Add the access token. return provider.get_auth_denied_response('token', client_id, redirect_uri, scope=scope) @web.route('/oauth/authorize', methods=['GET']) @no_cache @param_required('client_id') @param_required('redirect_uri') @param_required('scope') def request_authorization_code(): provider = FlaskAuthorizationProvider() response_type = request.args.get('response_type', 'code') client_id = request.args.get('client_id', None) redirect_uri = request.args.get('redirect_uri', None) scope = request.args.get('scope', None) if (not current_user.is_authenticated or not provider.validate_has_scopes(client_id, current_user.db_user().username, scope)): if not provider.validate_redirect_uri(client_id, redirect_uri): current_app = provider.get_application_for_client_id(client_id) if not current_app: abort(404) return provider._make_redirect_error_response(current_app.redirect_uri, 'redirect_uri_mismatch') # Load the scope information. scope_info = scopes.get_scope_information(scope) if not scope_info: abort(404) return # Load the application information. oauth_app = provider.get_application_for_client_id(client_id) app_email = oauth_app.avatar_email or oauth_app.organization.email oauth_app_view = { 'name': oauth_app.name, 'description': oauth_app.description, 'url': oauth_app.application_uri, 'avatar': json.dumps(avatar.get_data(oauth_app.name, app_email, 'app')), 'organization': { 'name': oauth_app.organization.username, 'avatar': json.dumps(avatar.get_data_for_org(oauth_app.organization)) } } # Show the authorization page. has_dangerous_scopes = any([check_scope['dangerous'] for check_scope in scope_info]) return render_page_template_with_routedata('oauthorize.html', scopes=scope_info, has_dangerous_scopes=has_dangerous_scopes, application=oauth_app_view, enumerate=enumerate, client_id=client_id, redirect_uri=redirect_uri, scope=scope, csrf_token_val=generate_csrf_token()) if response_type == 'token': return provider.get_token_response(response_type, client_id, redirect_uri, scope=scope) else: return provider.get_authorization_code(response_type, client_id, redirect_uri, scope=scope) @web.route('/oauth/access_token', methods=['POST']) @no_cache @param_required('grant_type', allow_body=True) @param_required('client_id', allow_body=True) @param_required('redirect_uri', allow_body=True) @param_required('code', allow_body=True) @param_required('scope', allow_body=True) def exchange_code_for_token(): grant_type = request.values.get('grant_type', None) client_id = request.values.get('client_id', None) client_secret = request.values.get('client_id', None) redirect_uri = request.values.get('redirect_uri', None) code = request.values.get('code', None) scope = request.values.get('scope', None) # Sometimes OAuth2 clients place the client id/secret in the Auth header. basic_header = parse_basic_auth(request.headers.get('Authorization')) if basic_header is not None: client_id = basic_header[0] or client_id client_secret = basic_header[1] or client_secret provider = FlaskAuthorizationProvider() return provider.get_token(grant_type, client_id, client_secret, redirect_uri, code, scope=scope) @web.route('/systemlogsarchive', methods=['GET']) @process_oauth @route_show_if(features.SUPER_USERS) @no_cache def download_logs_archive(): # Note: We cannot use the decorator here because this is a GET method. That being said, this # information is sensitive enough that we want the extra protection. verify_csrf() if SuperUserPermission().can(): archive_data = build_logs_archive(app) return Response(archive_data, mimetype="application/octet-stream", headers={"Content-Disposition": "attachment;filename=erlogs.tar.gz"}) abort(403) @web.route('/bitbucket/setup/', methods=['GET']) @require_session_login @parse_repository_name() @route_show_if(features.BITBUCKET_BUILD) def attach_bitbucket_trigger(namespace_name, repo_name): permission = AdministerRepositoryPermission(namespace_name, repo_name) if permission.can(): repo = model.repository.get_repository(namespace_name, repo_name) if not repo: msg = 'Invalid repository: %s/%s' % (namespace_name, repo_name) abort(404, message=msg) trigger = model.build.create_build_trigger(repo, BitbucketBuildTrigger.service_name(), None, current_user.db_user()) try: oauth_info = BuildTriggerHandler.get_handler(trigger).get_oauth_url() except TriggerProviderException: trigger.delete_instance() logger.debug('Could not retrieve Bitbucket OAuth URL') abort(500) config = { 'access_token': oauth_info['access_token'] } access_token_secret = oauth_info['access_token_secret'] model.build.update_build_trigger(trigger, config, auth_token=access_token_secret) return redirect(oauth_info['url']) abort(403) @web.route('/customtrigger/setup/', methods=['GET']) @require_session_login @parse_repository_name() def attach_custom_build_trigger(namespace_name, repo_name): permission = AdministerRepositoryPermission(namespace_name, repo_name) if permission.can(): repo = model.repository.get_repository(namespace_name, repo_name) if not repo: msg = 'Invalid repository: %s/%s' % (namespace_name, repo_name) abort(404, message=msg) trigger = model.build.create_build_trigger(repo, CustomBuildTrigger.service_name(), None, current_user.db_user()) repo_path = '%s/%s' % (namespace_name, repo_name) full_url = '%s%s%s' % (url_for('web.repository', path=repo_path), '?tab=builds&newtrigger=', trigger.uuid) logger.debug('Redirecting to full url: %s', full_url) return redirect(full_url) abort(403) @web.route('/') @no_cache @process_oauth @parse_repository_name(include_tag=True) @anon_protect def redirect_to_repository(namespace_name, repo_name, tag_name): permission = ReadRepositoryPermission(namespace_name, repo_name) is_public = model.repository.repository_is_public(namespace_name, repo_name) if request.args.get('ac-discovery', 0) == 1: return index('') if permission.can() or is_public: repo_path = '/'.join([namespace_name, repo_name]) return redirect(url_for('web.repository', path=repo_path, tab="tags", tag=tag_name)) abort(404) @web.route('/') @no_cache @process_oauth @anon_protect def redirect_to_namespace(namespace): user_or_org = model.user.get_user_or_org(namespace) if not user_or_org: abort(404) if user_or_org.organization: return redirect(url_for('web.org_view', path=namespace)) else: return redirect(url_for('web.user_view', path=namespace))